#1 Mapped Custom Talk entries to ENPCResidents for additional shop en…

…tries.

docs/version.json

@@ -1 +1 @@
-129
+148

src/APICrawler.js

@@ -50,7 +50,8 @@ module.exports = class APICrawler {
 
     const {
       columns,
-      filter
+      filter,
+      silent
     } = this.config;
 
     // Used to handle API timeouts gracefully.
@@ -67,7 +68,7 @@ module.exports = class APICrawler {
     const limit = pageIn === -1 ? 1 : limitValues[limitValueOffset];
     const page = pageIn === -1 ? 1 : (this.recordsProcessed / limit) + 1;
 
-    if (pageIn !== -1 && page === 1 && limitValueOffset === 0) {
+    if (pageIn !== -1 && page === 1 && limitValueOffset === 0 && !silent) {
       console.time(log);
       console.info(`Starting ${isPaginated ? '' : 'un'}paginated fetch of ${log}.`);
     }
@@ -130,8 +131,11 @@ module.exports = class APICrawler {
 
     // If the resource is not paginated, return the data.
     if (!isPaginated || !data.Pagination) {
-      console.info(`Finished unpaginated fetch of ${log}.`);
-      console.timeEnd(log);
+      if (!silent) {
+        console.info(`Finished unpaginated fetch of ${log}.`);
+        console.timeEnd(log);
+      }
+
       return data;
     }
 

src/config/data.js

@@ -19,6 +19,7 @@ const itemActionTypesQuery = (prefix) => [
 ];
 
 const customTalkScriptArgs = 30;
+const eNPCBaseDataRefs = 32;
 
 const fishingSpotItemIDFields = [
   'Item0TargetID',
@@ -366,6 +367,8 @@ module.exports = {
       cacheAs: 'npc-shops',
       /**
        * For ENPCResidents we need to extract the following fields:
+       * `Base` - The ENPCBase data's...
+       *   `ENpcData{0...n}` - External references used for mapping to Custom Talk entries;
        * `GilShop.*` - An array of gil shops attached to the NPC's...
        *   `Name_{lang}` - Localised name;
        *   `ID` - Shop identifier;
@@ -375,22 +378,15 @@ module.exports = {
        * `SpecialShop` - An array of special shops attached to the NPC.
        */
       columns: [
+        ...new Array(eNPCBaseDataRefs).fill(1).map((_, index) => `Base.ENpcData${index}`),
         ...helper.localisedColumnProperty(`GilShop.*.Name`),
         'GilShop.*.ID',
         'GilShop.*.Items',
         'ID',
         ...helper.localisedColumnProperty(`Name`),
         'SpecialShop'
       ],
-      /**
-       * Filter out any data sets which:
-       * 1. Have no GilShops.
-       */
-      filter: (data) => {
-        return data.filter(
-          eNPCResident => eNPCResident.GilShop.length || eNPCResident.SpecialShop.length
-        );
-      },
+      eNPCBaseDataRefs,
       isPaginated: true,
       log: 'ENPCResidents (for Shops)',
       method: 'fetch',

src/parsers/customTalk.js

@@ -1,5 +1,6 @@
 const APICrawler = require('../APICrawler');
 const fs = require('fs');
+const Progress = require('cli-progress');
 
 /**
  * Parse custom talk data from XIVAPI.
@@ -17,6 +18,9 @@ module.exports = async (data) => {
     specialShops: []
   };
 
+  const gilShops = [];
+  const specialShops = [];
+
   for (const entry of data) {
     for (let i = 0; i < customTalkScriptArgs; i++) {
       const reference = entry[`ScriptArg${i}`];
@@ -26,35 +30,73 @@ module.exports = async (data) => {
           args: [reference],
           isPaginated: false,
           log: `Gil Shop #${reference}`,
-          name: 'gilShop'
+          name: 'gilShop',
+          silent: true
         });
 
         const data = await crawler.fetch();
 
         // Gil shops.
-        parsed.gilShops.push({
+        gilShops.push({
           customTalk: entry.ID,
-          data
+          reference
         });
       } else if (reference >= 1760000 && reference < 1770000) {
-        const crawler = await new APICrawler({
-          args: [reference],
-          isPaginated: false,
-          log: `Special Shop #${reference}`,
-          name: 'specialShop'
-        });
-
-        const data = await crawler.fetch();
-
         // Special shops.
-        parsed.specialShops.push({
+        specialShops.push({
           customTalk: entry.ID,
-          data
+          reference
         });
       }
     }
   };
 
+  const progressBar = new Progress.Bar({}, Progress.Presets.shades_grey);
+  progressBar.start(gilShops.length + specialShops.length, 0);
+  let parsedCount = 0;
+
+  // Parse Gil Shops.
+  for (const entry of gilShops) {
+    const crawler = new APICrawler({
+      args: [entry.reference],
+      isPaginated: false,
+      log: `Gil Shop #${entry.reference}`,
+      name: 'gilShop',
+      silent: true
+    });
+
+    const data = await crawler.fetch();
+
+    parsed.gilShops.push({
+      customTalk: entry.customTalk,
+      data
+    });
+
+    progressBar.update(++parsedCount);
+  }
+
+  // Parse Special Shops.
+  for (const entry of specialShops) {
+    const crawler = await new APICrawler({
+      args: [entry.reference],
+      isPaginated: false,
+      log: `Special Shop #${entry.reference}`,
+      name: 'specialShop',
+      silent: true
+    });
+
+    const data = await crawler.fetch();
+
+    parsed.specialShops.push({
+      customTalk: entry.customTalk,
+      data
+    });
+
+    progressBar.update(++parsedCount);
+  }
+
+  progressBar.stop();
+
   fs.writeFileSync(
     './data/customTalk.json',
     JSON.stringify(parsed),

src/parsers/shops.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const currencies = require('../../data/currencies.json');
+const customTalk = require('../../data/customTalk.json');
 const currencyParser = require('../parsers/currencies');
 const items = require('../../data/items.json');
 
@@ -11,6 +12,12 @@ module.exports = (
   gcScripShopItems
 ) => {
   const config = require('../config/data').shops;
+
+  const {
+    eNPCBaseDataRefs,
+    specialShopItemIndexes
+  } = config.eNPCResident;
+
   let currenciesAddedCount = 0;
   const parsed = {};
 
@@ -21,9 +28,48 @@ module.exports = (
 
   const gil = currencies.find(currency => currency.id === 1);
 
+  // Extend data set to include Custom Talk Gil Shops and Special Shops.
+  eNPCResidents.filter(eNPCResident => {
+    const {
+      Base
+    } = eNPCResident;
+
+    if (!Base) {
+      return;
+    }
+
+    for (let i = 0; i < eNPCBaseDataRefs; i++) {
+      const v = Base[`ENpcData${i}`];
+
+      if (v >= 720000 && v < 730000) {
+        return true;
+      }
+    }
+  }).forEach(eNPCResident => {
+    for (let i = 0; i < eNPCBaseDataRefs; i++) {
+      const v = eNPCResident.Base[`ENpcData${i}`];
+
+      if (v >= 720000 && v < 730000) {
+        // Gil shops.
+        const customTalkGilShopMatches = customTalk.gilShops.filter(entry => entry.customTalk === v);
+
+        for (const customTalkGilShop of customTalkGilShopMatches) {
+          eNPCResident.GilShop.push(customTalkGilShop.data);
+        }
+
+        // Special shops.
+        const customTalkSpecialShopMatches = customTalk.specialShops.filter(entry => entry.customTalk === v);
+
+        for (const customTalkSpecialShop of customTalkSpecialShopMatches) {
+          eNPCResident.SpecialShop.push(customTalkSpecialShop.data);
+        }
+      }
+    }
+  });
+
   // Gil shops...
   eNPCResidents.filter(
-    eNPCResident => eNPCResident.GilShop.length
+    eNPCResident => Array.isArray(eNPCResident.GilShop) && eNPCResident.GilShop.length
   ).reduce((arr, eNPCResident) => ([
     ...arr,
     ...eNPCResident.GilShop.reduce((arr2, entry) => ([
@@ -70,13 +116,9 @@ module.exports = (
     })
   });
 
-  const {
-    specialShopItemIndexes
-  } = config.eNPCResident;
-
   // Special shops...
   eNPCResidents.filter(
-    eNPCResident => eNPCResident.SpecialShop.length
+    eNPCResident => Array.isArray(eNPCResident.SpecialShop) && eNPCResident.SpecialShop.length
   ).reduce((arr, eNPCResident) => ([
     ...arr,
     ...eNPCResident.SpecialShop.reduce((arr2, specialShop) => ([

src/update.js

@@ -172,8 +172,8 @@ class API {
       );
 
       // Custom Talk entries.
-      const customTalk = await this.crawl(config.customTalk);
-      await require('./parsers/customTalk')(customTalk);
+      // const customTalk = await this.crawl(config.customTalk);
+      // await require('./parsers/customTalk')(customTalk);
 
       // // Shops.
       const eNPCResidents = await this.crawl(config.shops.eNPCResident);