Merge pull request #4 from Chaphasilor/dev

bump to ODD 1.9.3.8

.gitignore

@@ -103,5 +103,9 @@ dist
 # TernJS port file
 .tern-port
 
+.vscode
+
 # ignore for now
 test.js
+
+ODD

README.md

@@ -10,6 +10,7 @@ Open Directory Downloader is a CLI tool for indexing so-called *Open Directories
 
 | Wrapper Version | Supported OpenDirectoryDownloader Versions | Included Version |
 | --- | --- | --- |
+| 2.0.0 | >1.9.3.5 | 1.9.3.8 |
 | 1.1.0 | >1.9.3.5 | 1.9.3.6 |
 | 1.0.0 | >1.9.3.1 | 1.9.3.3 |
 

config.js

@@ -1,10 +1,11 @@
-module.exports = {
-  GitHubReleasesUrl: `https://api.github.com/repos/KoalaBear84/OpenDirectoryDownloader/releases`,
-  OpenDirectoryDownloaderVersion: {
-    version: `v1.9.3.6`,
-    releaseId: `36853483`
-  },
-  OpenDirectoryDownloaderFolder: `${__dirname}/ODD`,
-  OpenDirectoryDownloaderOutputFolder: `${__dirname}/ODD/Scans`,
-  OpenDirectoryDownloaderPath: `${__dirname}/ODD/OpenDirectoryDownloader${process.platform === `win32` ? `.exe` : ``}`,
-}
+module.exports = {}
+
+module.exports.GitHubReleasesUrl = `https://api.github.com/repos/KoalaBear84/OpenDirectoryDownloader/releases`
+module.exports.OpenDirectoryDownloaderVersion = {
+  version: `v1.9.3.8`,
+  releaseId: `37128271`
+}
+module.exports.OpenDirectoryDownloaderFolder = `${__dirname}/ODD`
+module.exports.OpenDirectoryDownloaderOutputFolderName = `Scans`
+module.exports.OpenDirectoryDownloaderOutputFolder = `${module.exports.OpenDirectoryDownloaderFolder}/${module.exports.OpenDirectoryDownloaderOutputFolderName}`
+module.exports.OpenDirectoryDownloaderPath = `${module.exports.OpenDirectoryDownloaderFolder}/OpenDirectoryDownloader${process.platform === `win32` ? `.exe` : ``}`

install.js

@@ -1,3 +1,4 @@
+//TODO remove old files (not the whole ODD directory and not the Scans folder) before downloading the new version
 const fs = require(`fs`)
 
 const fetch = require(`node-fetch`)
@@ -7,7 +8,7 @@ const CONFIG = require(`./config`)
 const platform = process.platform
 const architecture = process.arch
 
-console.log(`Fetching release assets from GitHub...`)
+console.info(`Fetching release assets from GitHub...`)
 fetch(`${CONFIG.GitHubReleasesUrl}/${CONFIG.OpenDirectoryDownloaderVersion.releaseId}/assets`).then(res => res.json()).then(assets => {
 
   let releaseName
@@ -45,9 +46,39 @@ fetch(`${CONFIG.GitHubReleasesUrl}/${CONFIG.OpenDirectoryDownloaderVersion.relea
   let downloadUrl = asset.browser_download_url
 
   // console.log(`Creating ODD directory...`)
-  fs.mkdirSync(CONFIG.OpenDirectoryDownloaderFolder)
+
+  if (fs.existsSync(CONFIG.OpenDirectoryDownloaderFolder) && fs.lstatSync(CONFIG.OpenDirectoryDownloaderFolder).isDirectory()) {
+
+    console.info(`Removing old version of OpenDirectoryDownloader`)
+
+    const allDirents = fs.readdirSync(CONFIG.OpenDirectoryDownloaderFolder, {
+      withFileTypes: true,
+    })
+    const direntsToRemove = allDirents.filter(dirent => {
+      if (dirent.isDirectory()) {
+        return ![CONFIG.OpenDirectoryDownloaderOutputFolderName].includes(dirent.name)
+      } else {
+        return ![].includes(dirent.name)
+      }
+    })
+
+    for (const dirent of direntsToRemove) {
+
+      fs.rmSync(`${CONFIG.OpenDirectoryDownloaderFolder}/${dirent.name}`, {
+        recursive: true,
+      })
+
+    }
+
+  } else {
+    fs.mkdirSync(CONFIG.OpenDirectoryDownloaderFolder)
+  }
+
+  if (!fs.existsSync(CONFIG.OpenDirectoryDownloaderOutputFolder)) {
+    fs.mkdirSync(CONFIG.OpenDirectoryDownloaderOutputFolder)
+  }
 
-  console.log(`Starting download of OpenDirectoryDownloader executable...`)
+  console.info(`Starting download of OpenDirectoryDownloader executable...`)
   fetch(downloadUrl)
   .then(res => {
 
@@ -69,10 +100,10 @@ fetch(`${CONFIG.GitHubReleasesUrl}/${CONFIG.OpenDirectoryDownloaderVersion.relea
       if (platform === `linux`) {
         // make executable
         console.log(`Making binary executable...`)
-        fs.chmodSync(`${CONFIG.OpenDirectoryDownloaderFolder}/OpenDirectoryDownloader`, fs.constants.S_IXOTH)
+        fs.chmodSync(`${CONFIG.OpenDirectoryDownloaderFolder}/OpenDirectoryDownloader`, 0o755)
       }
 
-      console.log(`OpenDirectoryDownloader has been installed successfully to ${CONFIG.OpenDirectoryDownloaderFolder}!`)
+      console.info(`OpenDirectoryDownloader has been installed successfully to ${CONFIG.OpenDirectoryDownloaderFolder}!`)
 
     })
 

open-directory-downloader.js

@@ -1,4 +1,4 @@
-const { spawn } = require('child_process');
+const { exec } = require('child_process');
 const fs = require(`fs`);
 
 const CONFIG = require(`./config`)
@@ -9,6 +9,10 @@ module.exports = class OpenDirectoryDownloader {
 
     this.executable = executablePath;
     this.outputDir = outputDirectory;
+
+    if (!fs.existsSync(this.outputDir)) {
+      fs.mkdirSync(this.outputDir)
+    }
 
   }
 
@@ -42,7 +46,11 @@ module.exports = class OpenDirectoryDownloader {
         processArgs.push(`--output-file`)
         processArgs.push(options.outputFile)
       }
-      const oddProcess = spawn(this.executable, processArgs);
+
+      const oddProcess = exec(`${this.executable} ${processArgs.join(` `)}`, {
+        shell: true,
+        cwd: this.outputDir,
+      });
 
       let output = ``;
       let error = ``;
@@ -70,9 +78,16 @@ module.exports = class OpenDirectoryDownloader {
           reject(new Error(`ODD exited with code ${code}: ${error}`));
         }
 
+        console.log(`output:`, output)
+
         if (output.split(`Finished indexing`).length <= 1) {
           return reject(new Error(`ODD never finished indexing!`));
         }
+
+        if (output.split(`No URLs to save`).length > 1) {
+          // ODD found no files or subdirectories
+          return reject(new Error(`OpenDirectoryDownloader didn't find any files or directories on that site!`));
+        }
 
         // const finalResults = output.split(`Finished indexing`)[1];
         const finalResults = output.split(`Saving URL list to file..`)[1];

package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-directory-downloader",
-  "version": "1.1.0",
+  "version": "2.0.0",
   "description": "A NodeJS wrapper around KoalaBear84/OpenDirectoryDownloader",
   "main": "open-directory-downloader.js",
   "scripts": {