test: e2e tests fix, better logging (#1713)

- Fixes the e2e tests in regard to the new [apify.com](apify.com)
structure
- `stdout` + `stderr` from `execSync` in the e2e tests are UTF-8 decoded
now, no more byte arrays in error logs
- All e2e tests now have the new `.actor/actor.json` structure
- This was causing the PLATFORM issue, the new `apify-cli` version
automatically transforms the old config, but requires confirmation - no
interactive shell, no show, I guess 🤷🏻
  
TODO:
- random platform tests still failing, mostly because of request
timeouts - seems more like a platform issue though

test/e2e/automatic-persist-value/actor/apify.json → test/e2e/automatic-persist-value/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-automatic-persist-value",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/autoscaling-max-tasks-per-minute/actor/apify.json → test/e2e/autoscaling-max-tasks-per-minute/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-autoscaling-max-tasks-per-minute",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/autoscaling-max-tasks-per-minute/test.mjs

@@ -7,6 +7,6 @@ const { stats } = await runActor(testActorDirname);
 
 await expect(stats.requestsFinished === 2, 'All requests finished');
 await expect(
-    stats.crawlerRuntimeMillis > 60_000 && stats.crawlerRuntimeMillis < 65_000,
+    stats.crawlerRuntimeMillis > 60_000 && stats.crawlerRuntimeMillis < 70_000,
     `Ran one task per minute, took ~1 minute to complete, but no more than that (${stats.crawlerRuntimeMillis}ms)`,
 );

test/e2e/cheerio-default-ts/actor/apify.json → test/e2e/cheerio-default-ts/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-default-ts",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/cheerio-default/actor/apify.json → test/e2e/cheerio-default/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-default",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/cheerio-enqueue-links/actor/apify.json → test/e2e/cheerio-enqueue-links/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-enqueue-links",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/cheerio-ignore-ssl-errors/actor/apify.json → test/e2e/cheerio-ignore-ssl-errors/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-ignore-ssl-errors",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/cheerio-initial-cookies/actor/apify.json → test/e2e/cheerio-initial-cookies/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-initial-cookies",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/cheerio-max-requests/actor/apify.json → test/e2e/cheerio-max-requests/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-max-requests",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/cheerio-max-requests/actor/main.js

@@ -15,9 +15,9 @@ await Actor.main(async () => {
             const { url, userData: { label } } = request;
 
             if (label === 'START') {
-                const links = $('.ActorStoreItem').toArray().map((item) => $(item).attr('href'));
+                const links = $('a.card').toArray().map((item) => $(item).attr('href'));
                 for (const link of links) {
-                    const actorDetailUrl = `https://apify.com${link}`;
+                    const actorDetailUrl = `https://crawlee.dev${link}`;
                     await crawler.addRequests([{
                         url: actorDetailUrl,
                         userData: { label: 'DETAIL' },
@@ -26,21 +26,19 @@ await Actor.main(async () => {
             } else if (label === 'DETAIL') {
                 const uniqueIdentifier = url.split('/').slice(-2).join('/');
                 const title = $('header h1').text();
-                const description = $('header span.actor-description').text();
-                const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');
-                const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');
+                const firstParagraph = $('header + p').text();
+                const modifiedDate = $('.theme-last-updated time').attr('datetime');
 
                 await Dataset.pushData({
                     url,
                     uniqueIdentifier,
                     title,
-                    description,
-                    modifiedDate: new Date(Number(modifiedDate)),
-                    runCount: Number(runCount),
+                    firstParagraph,
+                    modifiedDate,
                 });
             }
         },
     });
 
-    await crawler.run([{ url: 'https://apify.com/apify', userData: { label: 'START' } }]);
+    await crawler.run([{ url: 'https://crawlee.dev/docs/examples', userData: { label: 'START' } }]);
 }, mainOptions);

test/e2e/cheerio-max-requests/test.mjs

@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname);
 await expect(stats.requestsFinished > 10, 'All requests finished');
 await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items');
 await expect(
-    validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),
+    validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'firstParagraph', 'modifiedDate']),
     'Dataset items validation',
 );

test/e2e/cheerio-page-info/actor/apify.json → test/e2e/cheerio-page-info/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-page-info",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/cheerio-page-info/actor/main.js

@@ -12,7 +12,7 @@ const router = createCheerioRouter();
 router.addHandler('START', async ({ enqueueLinks }) => {
     await enqueueLinks({
         label: 'DETAIL',
-        globs: ['https://apify.com/apify/web-scraper'],
+        globs: ['**/examples/accept-user-input'],
     });
 });
 
@@ -21,17 +21,15 @@ router.addHandler('DETAIL', async ({ request, $ }) => {
 
     const uniqueIdentifier = url.split('/').slice(-2).join('/');
     const title = $('header h1').text();
-    const description = $('header span.actor-description').text();
-    const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');
-    const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');
+    const firstParagraph = $('header + p').text();
+    const modifiedDate = $('.theme-last-updated time').attr('datetime');
 
     await Dataset.pushData({
         url,
         uniqueIdentifier,
         title,
-        description,
-        modifiedDate: new Date(Number(modifiedDate)),
-        runCount: Number(runCount),
+        firstParagraph,
+        modifiedDate,
     });
 });
 
@@ -40,5 +38,5 @@ await Actor.main(async () => {
         requestHandler: router,
     });
 
-    await crawler.run([{ url: 'https://apify.com/apify', userData: { label: 'START' } }]);
+    await crawler.run([{ url: 'https://crawlee.dev/docs/3.0/examples', userData: { label: 'START' } }]);
 }, mainOptions);

test/e2e/cheerio-page-info/test.mjs

@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname);
 await expect(stats.requestsFinished === 2, 'All requests finished');
 await expect(datasetItems.length === 1, 'Number of dataset items');
 await expect(
-    validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),
+    validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'firstParagraph', 'modifiedDate']),
     'Dataset items validation',
 );

test/e2e/cheerio-throw-on-ssl-errors/actor/apify.json → test/e2e/cheerio-throw-on-ssl-errors/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-cheerio-throw-on-ssl-errors",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/input-json5/actor/apify.json → test/e2e/input-json5/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-input-json5",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/jsdom-default-ts/actor/apify.json → test/e2e/jsdom-default-ts/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-jsdom-default-ts",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/jsdom-react-ts/actor/apify.json → test/e2e/jsdom-react-ts/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-jsdom-react-ts",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/migration/actor/apify.json → test/e2e/migration/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-migration",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/migration/actor/main.js

@@ -37,7 +37,7 @@ if (workerData !== '#actor') {
             maxRequestsPerCrawl: 5,
             async requestHandler({ enqueueLinks, request }) {
                 const { url } = request;
-                await enqueueLinks({ pseudoUrls: ['https://apify.com[(/[\\w-]+)?]'] });
+                await enqueueLinks();
 
                 await Dataset.pushData({ url });
 
@@ -48,6 +48,6 @@ if (workerData !== '#actor') {
         // eslint-disable-next-line no-underscore-dangle
         Configuration.getGlobalConfig().getStorageClient().__purged = false;
 
-        await crawler.run(['https://apify.com']);
+        await crawler.run(['https://crawlee.dev']);
     }, mainOptions);
 }

test/e2e/playwright-chromium-experimental-containers/actor/apify.json → test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-playwright-chromium-experimental-containers",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/playwright-default/actor/apify.json → test/e2e/playwright-default/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-playwright-default",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/playwright-default/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/playwright-default/actor/main.js

@@ -16,9 +16,11 @@ await Actor.main(async () => {
             const { url } = request;
             const pageTitle = await page.title();
             await Dataset.pushData({ url, pageTitle });
-            await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });
+            await enqueueLinks({
+                globs: ['**/3.0/examples/*'],
+            });
         },
     });
 
-    await crawler.run(['https://apify.com']);
+    await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);
 }, mainOptions);

test/e2e/playwright-default/test.mjs

@@ -5,6 +5,6 @@ await initialize(testActorDirname);
 
 const { stats, datasetItems } = await runActor(testActorDirname, 16384);
 
-await expect(stats.requestsFinished > 50, 'All requests finished');
-await expect(datasetItems.length > 50 && datasetItems.length < 150, 'Number of dataset items');
+await expect(stats.requestsFinished > 15, 'All requests finished');
+await expect(datasetItems.length > 15 && datasetItems.length < 25, 'Number of dataset items');
 await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');

test/e2e/playwright-enqueue-links/actor/apify.json → test/e2e/playwright-enqueue-links/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-playwright-enqueue-links",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/playwright-enqueue-links/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/playwright-firefox-experimental-containers/actor/apify.json → test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-playwright-firefox-experimental-containers",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/playwright-initial-cookies/actor/apify.json → test/e2e/playwright-initial-cookies/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-playwright-initial-cookies",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/playwright-initial-cookies/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/proxy-rotation/actor/apify.json → test/e2e/proxy-rotation/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-proxy-rotation",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/proxy-rotation/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/puppeteer-default/actor/apify.json → test/e2e/puppeteer-default/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-puppeteer-default",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/puppeteer-default/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/puppeteer-default/actor/main.js

@@ -16,9 +16,11 @@ await Actor.main(async () => {
             const { url } = request;
             const pageTitle = await page.title();
             await Dataset.pushData({ url, pageTitle });
-            await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });
+            await enqueueLinks({
+                globs: ['**/3.0/examples/*'],
+            });
         },
     });
 
-    await crawler.run(['https://apify.com']);
+    await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);
 }, mainOptions);

test/e2e/puppeteer-default/test.mjs

@@ -5,6 +5,6 @@ await initialize(testActorDirname);
 
 const { stats, datasetItems } = await runActor(testActorDirname, 16384);
 
-await expect(stats.requestsFinished > 50, 'All requests finished');
-await expect(datasetItems.length > 50 && datasetItems.length < 150, 'Number of dataset items');
+await expect(stats.requestsFinished > 15, 'All requests finished');
+await expect(datasetItems.length > 15 && datasetItems.length < 25, 'Number of dataset items');
 await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');

test/e2e/puppeteer-enqueue-links/actor/apify.json → test/e2e/puppeteer-enqueue-links/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-puppeteer-enqueue-links",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/puppeteer-enqueue-links/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/puppeteer-ignore-ssl-errors/actor/apify.json → test/e2e/puppeteer-ignore-ssl-errors/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-puppeteer-ignore-ssl-errors",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \

test/e2e/puppeteer-initial-cookies/actor/apify.json → test/e2e/puppeteer-initial-cookies/actor/.actor/actor.json

@@ -1,4 +1,5 @@
 {
+	"actorSpecification": 1,
 	"name": "test-puppeteer-initial-cookies",
 	"version": "0.0",
 	"buildTag": "latest",

test/e2e/puppeteer-initial-cookies/actor/Dockerfile

@@ -12,7 +12,7 @@ RUN rm -r node_modules
 COPY --from=builder /node_modules ./node_modules
 COPY --from=builder /packages ./packages
 COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
 COPY /main.js ./
 
 RUN echo "Installed NPM packages:" \