Move getArticleIds to mw-api, minor fixes

src/MediaWiki.ts

@@ -29,7 +29,7 @@ class MediaWiki {
   public getCategories: boolean
   public namespaces: MWNamespaces = {}
   public namespacesToMirror: string[] = []
-  public mwArticleId: string
+  public apiCheckArticleId: string
 
   #wikiPath: string
   #restApiPath: string
@@ -95,7 +95,7 @@ class MediaWiki {
 
     this.#apiPath = 'w/api.php'
     this.#wikiPath = 'wiki/'
-    this.mwArticleId = 'MediaWiki:Sidebar'
+    this.apiCheckArticleId = 'MediaWiki:Sidebar'
 
     this.#hasWikimediaDesktopRestApi = null
     this.#hasVisualEditorApi = null
@@ -107,15 +107,15 @@ class MediaWiki {
 
   public async hasWikimediaDesktopRestApi(): Promise<boolean> {
     if (this.#hasWikimediaDesktopRestApi === null) {
-      this.#hasWikimediaDesktopRestApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.mwArticleId))
+      this.#hasWikimediaDesktopRestApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
       return this.#hasWikimediaDesktopRestApi
     }
     return this.#hasWikimediaDesktopRestApi
   }
 
   public async hasVisualEditorApi(): Promise<boolean> {
     if (this.#hasVisualEditorApi === null) {
-      this.#hasVisualEditorApi = await checkApiAvailability(this.visualEditorURLDirector.buildArticleURL(this.mwArticleId))
+      this.#hasVisualEditorApi = await checkApiAvailability(this.visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
       return this.#hasVisualEditorApi
     }
     return this.#hasVisualEditorApi

src/mwoffliner.lib.ts

@@ -49,7 +49,7 @@ import { Dump } from './Dump.js'
 import { config } from './config.js'
 import MediaWiki from './MediaWiki.js'
 import Downloader from './Downloader.js'
-import { getArticleIds } from './util/redirects.js'
+import { getArticleIds } from './util/mw-api.js'
 import { articleListHomeTemplate } from './Templates.js'
 import { downloadFiles, saveArticles } from './util/saveArticles.js'
 import { getCategoriesForArticles, trimUnmirroredPages } from './util/categories.js'
@@ -208,7 +208,7 @@ async function execute(argv: any) {
     }
   }
 
-  MediaWiki.mwArticleId = mwMetaData.mainPage
+  MediaWiki.apiCheckArticleId = mwMetaData.mainPage
   await MediaWiki.hasWikimediaDesktopRestApi()
   await MediaWiki.hasVisualEditorApi()
 

src/util/builders/url/basic.director.ts

@@ -3,7 +3,7 @@ import urlBuilder from './url.builder.js'
 type DownloaderBaseUrlConditions = Array<{ condition: boolean; value: string }>
 
 /**
- * Build base URL for specific wiki. Expamle of the output: 'https://en.wikipedia.org' or 'https://de.wikivoyage.org
+ * Build base URL for specific wiki. Example of the output: 'https://en.wikipedia.org' or 'https://de.wikivoyage.org
  */
 class BasicURLDirector {
   buildMediawikiBaseURL(domain: string) {

src/util/mw-api.ts

@@ -4,6 +4,7 @@ import * as logger from '../Logger.js'
 import Downloader from '../Downloader.js'
 import Timer from './Timer.js'
 import axios from 'axios'
+import MediaWiki from '../MediaWiki.js'
 
 export async function getArticlesByIds(articleIds: string[], downloader: Downloader, redisStore: RS, log = true): Promise<void> {
   let from = 0
@@ -263,3 +264,21 @@ export async function checkApiAvailability(url: string, loginCookie = ''): Promi
     return false
   }
 }
+
+export async function getArticleIds(downloader: Downloader, redisStore: RS, mainPage?: string, articleIds?: string[], articleIdsToIgnore?: string[]) {
+  if (mainPage) {
+    await getArticlesByIds([mainPage], downloader, redisStore)
+  }
+
+  if (articleIds) {
+    await getArticlesByIds(articleIds, downloader, redisStore)
+  } else {
+    await pmap(
+      MediaWiki.namespacesToMirror,
+      (namespace: string) => {
+        return getArticlesByNS(MediaWiki.namespaces[namespace].num, downloader, redisStore, articleIdsToIgnore)
+      },
+      { concurrency: downloader.speed },
+    )
+  }
+}

test/unit/mwApi.test.ts

@@ -1,7 +1,7 @@
 import { startRedis, stopRedis, redisStore } from './bootstrap.js'
 import Downloader from '../../src/Downloader.js'
 import MediaWiki from '../../src/MediaWiki.js'
-import { getArticleIds } from '../../src/util/redirects.js'
+import { getArticleIds } from '../../src/util/mw-api.js'
 import { getArticlesByNS } from '../../src/util/index.js'
 import { config } from '../../src/config.js'
 import { jest } from '@jest/globals'

test/unit/renderers/article.renderer.test.ts

@@ -17,12 +17,10 @@ describe('ArticleRenderer', () => {
     const visualEditorRenderer = new VisualEditorRenderer()
 
     it('should throw if no JSON was provided', async () => {
-      const renderOpts = {
-        data: null,
-      }
-
       expect(async () => {
-        await visualEditorRenderer.render(renderOpts)
+        await visualEditorRenderer.render({
+          data: null,
+        })
       }).rejects.toThrow(new Error('Cannot render [null] into an article'))
     })
 

test/unit/saveArticles.test.ts

@@ -199,13 +199,11 @@ describe('saveArticles', () => {
     }
     const visualEditorRenderer = new VisualEditorRenderer()
 
-    const renderOpts = {
-      data: articleJsonObject,
-      articleId: 'deletedArticle',
-    }
-
     expect(async () => {
-      await visualEditorRenderer.render(renderOpts)
+      await visualEditorRenderer.render({
+        data: articleJsonObject,
+        articleId: 'deletedArticle',
+      })
     }).rejects.toThrow(new Error(DELETED_ARTICLE_ERROR))
   })
 

test/unit/urlRewriting.test.ts

@@ -2,7 +2,7 @@ import { startRedis, stopRedis, redisStore } from './bootstrap.js'
 import domino from 'domino'
 import { rewriteUrl } from '../../src/util/rewriteUrls.js'
 import { makeLink, setupScrapeClasses } from '../util.js'
-import { getArticleIds } from '../../src/util/redirects.js'
+import { getArticleIds } from '../../src/util/mw-api.js'
 import { saveArticles } from '../../src/util/saveArticles.js'
 import { ZimArticle } from '@openzim/libzim'
 import { mwRetToArticleDetail } from '../../src/util/index.js'