From eab278a724a9951586778245cd3e0c0ca3c279f5 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 24 Aug 2026 16:09:22 +0530 Subject: [PATCH 1/3] fix(CMG-1109): show content-embedded WordPress assets on Map Entry Assets extractAssets only read formal wp:post_type=attachment items, so images embedded in post content (no attachment item backing them) never got a row on the Map Entry Assets screen, even though the actual migration downloads and creates them as real Contentstack assets via wordpress.service.ts's content:encoded scan. Extends extraction to find those images too, matching the id scheme the real migration uses so uids still resolve, and dedupes against formal attachment items so an overlapping URL doesn't get two rows. --- .../migration-wordpress/libs/extractAssets.ts | 122 +++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/upload-api/migration-wordpress/libs/extractAssets.ts b/upload-api/migration-wordpress/libs/extractAssets.ts index 0136c36de..ef5b2d842 100644 --- a/upload-api/migration-wordpress/libs/extractAssets.ts +++ b/upload-api/migration-wordpress/libs/extractAssets.ts @@ -1,4 +1,5 @@ import fs from 'fs'; +import * as cheerio from 'cheerio'; export interface AssetMappingRow { id: string; @@ -52,14 +53,100 @@ const getTitle = (item: any, filename: string): string => { return filename.split('.').slice(0, -1).join('.') || filename; }; +// Mirrors wordpress.service.ts's isValidImageUrl — keep in sync. +const isValidImageUrl = (url: string): boolean => { + if (!url || typeof url !== 'string') return false; + if (url.trim().startsWith('data:')) return false; + if (url.trim().length < 5) return false; + const lowerUrl = url.toLowerCase().trim(); + if (lowerUrl.startsWith('javascript:') || lowerUrl.startsWith('mailto:') || lowerUrl.startsWith('tel:')) { + return false; + } + return true; +}; + +// Mirrors wordpress.service.ts's toCheckUrl — keep in sync. +const toCheckUrl = (url: string, baseSiteUrl: string): string => { + const validPattern = /^(https?:\/\/|www\.)/; + return validPattern.test(url) ? url : `${baseSiteUrl}${url.replace(/^\/+/, '')}`; +}; + +/** + * Finds embedded image URLs in a post's content:encoded, covering the img + * src/data-src/srcset cases — the common WordPress block-editor patterns. + * Mirrors (a practical subset of) wordpress.service.ts's + * extractImageUrlsFromContent, which is what the actual migration run scans + * to decide which content-embedded images to download. Keep in sync: if that + * function's matching rules grow, this should too, or rows will exist here + * for images the real run doesn't find (or vice versa). + */ +const extractImageUrlsFromContent = (htmlContent: string, baseSiteUrl: string): string[] => { + if (!htmlContent || typeof htmlContent !== 'string') return []; + const imageUrls = new Set(); + try { + const $ = cheerio.load(htmlContent); + $('img').each((_, element) => { + const el = $(element); + const src = el.attr('src'); + if (src && isValidImageUrl(src)) { + const fullUrl = toCheckUrl(src, baseSiteUrl); + if (isValidImageUrl(fullUrl)) imageUrls.add(fullUrl); + } + const dataSrc = el.attr('data-src'); + if (dataSrc && isValidImageUrl(dataSrc)) { + const fullUrl = toCheckUrl(dataSrc, baseSiteUrl); + if (isValidImageUrl(fullUrl)) imageUrls.add(fullUrl); + } + const srcset = el.attr('srcset'); + if (srcset) { + srcset.split(',').map((s) => s.trim().split(/\s+/)[0]).forEach((url) => { + if (isValidImageUrl(url)) { + const fullUrl = toCheckUrl(url, baseSiteUrl); + if (isValidImageUrl(fullUrl)) imageUrls.add(fullUrl); + } + }); + } + }); + } catch { + // Malformed content:encoded — treat as no embedded images rather than failing extraction. + } + return Array.from(imageUrls); +}; + +/** + * Derives the same {uid, filename, title} a content-embedded image gets when + * the real migration run downloads it via wordpress.service.ts's + * saveAssetFromUrl. That function's customId (filename without extension, + * dashes to underscores, lowercased) is what ends up as the key in + * uid-mapping.json — otherCmsAssetUid must match it verbatim or the row can + * never resolve a Contentstack uid, the same class of bug fixed for formal + * attachment items. Unlike those, this is NOT prefixed with `assets_`. + */ +const parseContentAssetUrl = (url: string): { uid: string; filename: string; title: string } | null => { + const originalName = url.split('/').pop()?.split('?')[0] || ''; + if (!originalName) return null; + const nameWithoutExt = originalName.includes('.') + ? originalName.substring(0, originalName.lastIndexOf('.')) + : originalName; + const uid = nameWithoutExt.replace(/-/g, '_').toLowerCase(); + if (!uid) return null; + return { uid, filename: originalName, title: nameWithoutExt }; +}; + const extractAssets = async (filePath: string): Promise => { const rows: AssetMappingRow[] = []; try { const rawData = await fs.promises.readFile(filePath, 'utf8'); const jsonData = JSON.parse(rawData); const items = normalizeArray(jsonData?.rss?.channel?.item); + const baseSiteUrl = jsonData?.rss?.channel?.['wp:base_site_url'] ?? jsonData?.channel?.['wp:base_site_url'] ?? ''; const seenIds = new Set(); + // Absolute URLs already represented by a formal attachment item — skip these when scanning + // content so the same picture doesn't get a second row (it would also become a second, + // duplicate Contentstack asset on the actual migration run — a pre-existing issue in + // getAllAssets this extraction shouldn't compound). + const attachmentUrls = new Set(); for (const item of items) { if (item?.['wp:post_type'] !== 'attachment') { @@ -76,6 +163,10 @@ const extractAssets = async (filePath: string): Promise => { const filename = getFilename(item, assetPath); const title = getTitle(item, filename); + if (assetPath) { + attachmentUrls.add(toCheckUrl(assetPath, baseSiteUrl)); + } + rows.push({ id, // Must match the `assets_` key wordpress.service.ts uses as the @@ -90,6 +181,35 @@ const extractAssets = async (filePath: string): Promise => { }); } + // Images embedded in post content but never declared as a formal attachment item still get + // migrated (wordpress.service.ts's getAllAssets scans content:encoded independently of the + // attachment-item pass) — without this, the Map Entry Assets screen never had a row for them + // at all, on any iteration, even though they exist as real Contentstack assets afterward. + const seenContentUids = new Set(); + for (const item of items) { + const contentEncoded = item?.['content:encoded']; + if (!contentEncoded || typeof contentEncoded !== 'string') continue; + + const imageUrls = extractImageUrlsFromContent(contentEncoded, baseSiteUrl); + for (const url of imageUrls) { + if (attachmentUrls.has(url)) continue; + + const parsed = parseContentAssetUrl(url); + if (!parsed || seenContentUids.has(parsed.uid)) continue; + seenContentUids.add(parsed.uid); + + rows.push({ + id: parsed.uid, + otherCmsAssetUid: parsed.uid, + filename: parsed.filename, + title: parsed.title, + file_size: '', + assetPath: url, + isUpdate: false, + }); + } + } + return rows; } catch (error: any) { console.error('Error while extracting WordPress assets:', error?.message || error); @@ -97,4 +217,4 @@ const extractAssets = async (filePath: string): Promise => { } }; -export default extractAssets; \ No newline at end of file +export default extractAssets; From 651d00f47a432a9eb91b2dc48c409b6cc638f2ab Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 24 Aug 2026 16:09:23 +0530 Subject: [PATCH 2/3] fix(CMG-1004): lock displayed file format to the selected CMS's format Editing the file path after selecting a CMS with a single fixed allowed format (e.g. Sitecore, always Zip) re-derived the displayed "File Format" from the new path's extension, showing e.g. "XML" instead of staying "Zip". Only CMS types with more than one allowed format (currently just stack-to-stack Contentstack) should have the label follow the actual uploaded extension; everyone else keeps their fixed format, with the existing validation effect flagging a real mismatch instead. --- .../LegacyCms/Actions/LoadFileFormat.tsx | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx b/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx index 98453bc9c..be388e5a9 100644 --- a/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx @@ -50,12 +50,38 @@ const LoadFileFormat = (_props: LoadFileFormatProps) => { }, [newMigrationData]); // Handle file format extraction - RUN IMMEDIATELY ON MOUNT AND WHENEVER THE FILE PATH CHANGES. - // The displayed format is always derived from the ACTUAL uploaded file extension, never from a - // stale selectedFileFormat (which gets pre-seeded to the CMS default on CMS selection). This is - // why editing the file path (e.g. zip → json) now updates the label, icon, and Redux in sync. + // Most CMS types have exactly one allowed format (e.g. Sitecore is always Zip) — for those, the + // displayed format must stay locked to that fixed format regardless of what extension the user + // types in the path; the separate validation effect below already flags a mismatched upload. + // Only when the selected CMS allows more than one format (currently just stack-to-stack + // Contentstack, which accepts JSON or Zip) do we derive the displayed format from the actual + // uploaded file extension, since there's genuinely more than one valid answer to show. useEffect(() => { const filePath = newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath || ''; const currentFormat = newMigrationData?.legacy_cms?.selectedFileFormat?.title; + const allowedFormats = newMigrationData?.legacy_cms?.selectedCms?.allowed_file_formats; + + if (!validateArray(allowedFormats) || allowedFormats.length <= 1) { + const fixedFormat = allowedFormats?.[0]; + if (fixedFormat) { + setFileIcon(fixedFormat?.title); + setFileDisplayTitle(getDisplayTitle(fixedFormat?.title)); + if (newMigrationData?.legacy_cms?.selectedFileFormat?.fileformat_id?.toLowerCase() !== fixedFormat?.fileformat_id?.toLowerCase()) { + const latest = newMigrationDataRef.current; + dispatch(updateNewMigrationData({ + ...latest, + legacy_cms: { + ...latest?.legacy_cms, + selectedFileFormat: fixedFormat + } + })); + } + } else if (!isEmptyString(currentFormat)) { + setFileIcon(currentFormat); + setFileDisplayTitle(getDisplayTitle(currentFormat)); + } + return; + } // No file yet — fall back to whatever format is already in Redux (e.g. SQL/directory CMS types // that don't carry a localPath). @@ -110,7 +136,8 @@ const LoadFileFormat = (_props: LoadFileFormatProps) => { }, [ newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath, newMigrationData?.legacy_cms?.selectedFileFormat?.fileformat_id, - newMigrationData?.legacy_cms?.selectedFileFormat?.title + newMigrationData?.legacy_cms?.selectedFileFormat?.title, + newMigrationData?.legacy_cms?.selectedCms?.allowed_file_formats ]); // Validate the uploaded file's format against the selected CMS's allowed formats. From c1302d4a0608543919ea4b8ef78aad3a3bb640a2 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Mon, 24 Aug 2026 17:04:51 +0530 Subject: [PATCH 3/3] fix: address PR review feedback on #1150 - extractAssets.ts: cover a[href] image links, audio/source, and CSS background-image (inline +