Skip to content

Commit 73a9650

Browse files
paulpopuspaulpopus
authored
fix(plugin-import-export): errors when import/export files were stored in a storage adapter such as S3 (#15441)
Relies on #15405 first When using a storage adapter such as S3 to handle files there were errors in the way the files would be fetched for imports. This PR fixes that, adding test coverage and changes the job behaviour so that it only stores the import doc ID and not the full file data - this avoids many problems down the line and optimises memory and storage usage. Fixes #15247 Previously attempting to use an import with the jobs queue with S3 storage would result in this error: ```text [11:46:25] ERROR: Failed to queue import job for document 698628f19aaca7d4f4f18c6e err: { "type": "Error", "message": "ENOENT: no such file or directory, open '/posts-with-s3-import/2026-02-06 114542.csv'", "stack": Error: ENOENT: no such file or directory, open '/posts-with-s3-import/2026-02-06 114542.csv' ``` Also: - Exports `getExternalFile` utility from `payload/internals`
1 parent 60c65ed commit 73a9650

18 files changed

Lines changed: 980 additions & 586 deletions

File tree

‎.github/workflows/main.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ jobs:
332332

333333
- name: Start LocalStack
334334
run: pnpm docker:start
335-
if: matrix.suite == 'plugin-cloud-storage'
335+
if: contains(fromJson('["plugin-cloud-storage", "plugin-import-export"]'), matrix.suite)
336336

337337
- name: Start database
338338
id: db
@@ -513,7 +513,7 @@ jobs:
513513

514514
- name: Start LocalStack
515515
run: pnpm docker:start
516-
if: matrix.suite == 'plugin-cloud-storage'
516+
if: contains(fromJson('["plugin-cloud-storage", "plugin-import-export"]'), matrix.suite)
517517

518518
- name: Store Playwright's Version
519519
run: |

‎packages/payload/src/exports/internal.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
* Modules exported here are not part of the public API and are subject to change without notice and without a major version bump.
33
*/
44

5+
export { getExternalFile } from '../uploads/getExternalFile.js'
56
export { getRangeRequestInfo } from '../uploads/getRangeRequestInfo.js'
67
export { getSafeFileName } from '../uploads/getSafeFilename.js'
78
export { parseRangeHeader } from '../uploads/parseRangeHeader.js'

‎packages/plugin-import-export/src/components/CollectionField/index.tsx‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,26 @@ import { useEffect } from 'react'
77
import { useImportExport } from '../ImportExportProvider/index.js'
88

99
export const CollectionField: React.FC = () => {
10-
const { id, collectionSlug } = useDocumentInfo()
10+
const { id, collectionSlug, docConfig } = useDocumentInfo()
1111
const { setValue } = useField({ path: 'collectionSlug' })
1212
const { collection } = useImportExport()
1313

14+
const defaultCollectionSlug = docConfig?.admin?.custom?.defaultCollectionSlug as
15+
| string
16+
| undefined
17+
1418
useEffect(() => {
1519
if (id) {
1620
return
1721
}
1822
if (collection) {
1923
setValue(collection)
24+
} else if (defaultCollectionSlug) {
25+
setValue(defaultCollectionSlug)
2026
} else if (collectionSlug) {
2127
setValue(collectionSlug)
2228
}
23-
}, [id, collection, setValue, collectionSlug])
29+
}, [id, collection, setValue, collectionSlug, defaultCollectionSlug])
2430

2531
return null
2632
}

‎packages/plugin-import-export/src/export/createExport.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ export type Export = {
2727
*/
2828
debug?: boolean
2929
drafts?: 'no' | 'yes'
30-
exportsCollection: string
30+
exportCollection: string
3131
fields?: string[]
3232
format: 'csv' | 'json'
3333
globals?: string[]
@@ -65,7 +65,7 @@ export const createExport = async (args: CreateExportArgs) => {
6565
debug = false,
6666
download,
6767
drafts: draftsFromInput,
68-
exportsCollection,
68+
exportCollection,
6969
fields,
7070
format,
7171
limit: incomingLimit,
@@ -513,7 +513,7 @@ export const createExport = async (args: CreateExportArgs) => {
513513
}
514514
await req.payload.update({
515515
id,
516-
collection: exportsCollection,
516+
collection: exportCollection,
517517
data: {},
518518
file: {
519519
name,

‎packages/plugin-import-export/src/export/getCreateExportCollectionTask.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ export const getCreateCollectionExportTask = (
3131
type: 'text',
3232
},
3333
{
34-
name: 'exportsCollection',
34+
name: 'exportCollection',
3535
type: 'text',
3636
},
3737
{

‎packages/plugin-import-export/src/export/getExportCollection.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ export const getExportCollection = ({
101101
...exportData,
102102
batchSize,
103103
debug,
104-
exportsCollection: collectionConfig.slug,
104+
exportCollection: collectionConfig.slug,
105105
maxLimit,
106106
req,
107107
userCollection: user?.collection || user?.user?.collection,
@@ -130,7 +130,7 @@ export const getExportCollection = ({
130130
const input: Export = {
131131
...doc,
132132
batchSize,
133-
exportsCollection: collectionConfig.slug,
133+
exportCollection: collectionConfig.slug,
134134
maxLimit,
135135
userCollection: user?.collection || user?.user?.collection,
136136
userID: user?.id || user?.user?.id,

‎packages/plugin-import-export/src/import/batchProcessor.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,8 @@ async function processImportBatch({
188188
})
189189
}
190190

191-
delete createData._status // Remove _status from data - it's controlled via draft option
191+
// Remove _status from data - it's controlled via draft option
192+
delete createData._status
192193
}
193194

194195
if (req.payload.config.debug && 'title' in createData) {
@@ -452,7 +453,8 @@ async function processImportBatch({
452453
const statusValue = createData._status || options.defaultVersionStatus
453454
const isPublished = statusValue !== 'draft'
454455
draftOption = !isPublished
455-
delete createData._status // Remove _status from data - it's controlled via draft option
456+
// Remove _status from data - it's controlled via draft option
457+
delete createData._status
456458
}
457459

458460
// Check if we have multi-locale data and extract it

‎packages/plugin-import-export/src/import/getCreateImportCollectionTask.ts‎

Lines changed: 139 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,116 +1,168 @@
1-
import type { Config, TaskConfig, TypedUser } from 'payload'
1+
import type { Config, TaskConfig } from 'payload'
22

3-
import type { Import } from './createImport.js'
3+
import { FileRetrievalError } from 'payload'
44

5+
import { getFileFromDoc } from '../utilities/getFileFromDoc.js'
56
import { createImport } from './createImport.js'
6-
import { getFields } from './getFields.js'
77

88
export type ImportTaskInput = {
9+
batchSize?: number
10+
debug?: boolean
911
defaultVersionStatus?: 'draft' | 'published'
10-
importId?: string
11-
importsCollection?: string
12-
user?: string
13-
} & Import
12+
importCollection: string
13+
importId: string
14+
maxLimit?: number
15+
userCollection?: string
16+
userID?: number | string
17+
}
1418

1519
export const getCreateCollectionImportTask = (
16-
config: Config,
20+
_config: Config,
1721
): TaskConfig<{
1822
input: ImportTaskInput
1923
output: object
2024
}> => {
21-
const inputSchema = getFields(config).concat(
22-
{
23-
name: 'user',
24-
type: 'text',
25-
},
26-
{
27-
name: 'userCollection',
28-
type: 'text',
29-
},
30-
{
31-
name: 'importsCollection',
32-
type: 'text',
33-
},
34-
{
35-
name: 'file',
36-
type: 'group',
37-
fields: [
38-
{
39-
name: 'data',
40-
type: 'text',
41-
},
42-
{
43-
name: 'mimetype',
44-
type: 'text',
45-
},
46-
{
47-
name: 'name',
48-
type: 'text',
49-
},
50-
],
51-
},
52-
{
53-
name: 'format',
54-
type: 'select',
55-
options: ['csv', 'json'],
56-
},
57-
{
58-
name: 'debug',
59-
type: 'checkbox',
60-
},
61-
{
62-
name: 'maxLimit',
63-
type: 'number',
64-
},
65-
)
66-
6725
return {
6826
slug: 'createCollectionImport',
6927
handler: async ({ input, req }) => {
70-
// Convert file data back to Buffer if it was serialized
71-
if (input.file && typeof input.file.data === 'string') {
72-
input.file.data = Buffer.from(input.file.data, 'base64')
28+
const {
29+
batchSize,
30+
debug,
31+
defaultVersionStatus,
32+
importCollection,
33+
importId,
34+
maxLimit,
35+
userCollection,
36+
userID,
37+
} = input
38+
39+
// Fetch the import document to get all necessary data
40+
const importDoc = await req.payload.findByID({
41+
id: importId,
42+
collection: importCollection,
43+
})
44+
45+
if (!importDoc) {
46+
throw new Error(`Import document not found: ${importId}`)
47+
}
48+
49+
// Get the collection config for the imports collection
50+
const collectionConfig = req.payload.config.collections.find(
51+
(c) => c.slug === importCollection,
52+
)
53+
54+
if (!collectionConfig) {
55+
throw new Error(`Collection config not found for: ${importCollection}`)
56+
}
57+
58+
// Retrieve the file using getFileFromDoc (handles both local and cloud storage)
59+
const file = await getFileFromDoc({
60+
collectionConfig,
61+
doc: {
62+
filename: importDoc.filename as string,
63+
mimeType: importDoc.mimeType as string | undefined,
64+
url: importDoc.url as string | undefined,
65+
},
66+
req,
67+
})
68+
69+
const fileMimetype = file.mimetype || (importDoc.mimeType as string)
70+
71+
if (!fileMimetype) {
72+
throw new FileRetrievalError(
73+
req.t,
74+
`Unable to determine mimetype for file: ${importDoc.filename}`,
75+
)
7376
}
7477

7578
const result = await createImport({
76-
...input,
79+
name: (importDoc.filename as string) || 'import',
80+
batchSize,
81+
collectionSlug: importDoc.collectionSlug as string,
82+
debug,
83+
defaultVersionStatus,
84+
file: {
85+
name: importDoc.filename as string,
86+
data: file.data,
87+
mimetype: fileMimetype,
88+
},
89+
format: fileMimetype === 'text/csv' ? 'csv' : 'json',
90+
importMode: (importDoc.importMode as 'create' | 'update' | 'upsert') || 'create',
91+
matchField: importDoc.matchField as string | undefined,
92+
maxLimit,
7793
req,
94+
userCollection,
95+
userID,
7896
})
7997

80-
// Update the import document with results if importId is provided
81-
if (input.importId) {
82-
await req.payload.update({
83-
id: input.importId,
84-
collection: input.importsCollection || 'imports',
85-
data: {
86-
status:
87-
result.errors.length === 0
88-
? 'completed'
89-
: result.imported + result.updated === 0
90-
? 'failed'
91-
: 'partial',
92-
summary: {
93-
imported: result.imported,
94-
issueDetails:
95-
result.errors.length > 0
96-
? result.errors.map((e) => ({
97-
data: e.doc,
98-
error: e.error,
99-
row: e.index + 1,
100-
}))
101-
: undefined,
102-
issues: result.errors.length,
103-
total: result.total,
104-
updated: result.updated,
105-
},
98+
// Update the import document with results
99+
await req.payload.update({
100+
id: importId,
101+
collection: importCollection,
102+
data: {
103+
status:
104+
result.errors.length === 0
105+
? 'completed'
106+
: result.imported + result.updated === 0
107+
? 'failed'
108+
: 'partial',
109+
summary: {
110+
imported: result.imported,
111+
issueDetails:
112+
result.errors.length > 0
113+
? result.errors.map((e) => ({
114+
data: e.doc,
115+
error: e.error,
116+
row: e.index + 1,
117+
}))
118+
: undefined,
119+
issues: result.errors.length,
120+
total: result.total,
121+
updated: result.updated,
106122
},
107-
})
108-
}
123+
},
124+
})
109125

110126
return {
111127
output: result,
112128
}
113129
},
114-
inputSchema,
130+
inputSchema: [
131+
{
132+
name: 'importId',
133+
type: 'text',
134+
required: true,
135+
},
136+
{
137+
name: 'importCollection',
138+
type: 'text',
139+
required: true,
140+
},
141+
{
142+
name: 'userID',
143+
type: 'text',
144+
},
145+
{
146+
name: 'userCollection',
147+
type: 'text',
148+
},
149+
{
150+
name: 'batchSize',
151+
type: 'number',
152+
},
153+
{
154+
name: 'debug',
155+
type: 'checkbox',
156+
},
157+
{
158+
name: 'defaultVersionStatus',
159+
type: 'select',
160+
options: ['draft', 'published'],
161+
},
162+
{
163+
name: 'maxLimit',
164+
type: 'number',
165+
},
166+
],
115167
}
116168
}

0 commit comments

Comments
 (0)