Package hallucination

garak/configs/cutoff.yaml

@@ -0,0 +1,43 @@
+---
+system:
+    verbose: 0
+    narrow_output: false
+    parallel_requests: 16
+    parallel_attempts: 16
+    lite: true
+    show_z: false
+
+run:
+    seed:
+    deprefix: true
+    eval_threshold: 0.5
+    generations: 5
+    probe_tags:
+
+plugins:
+    model_type: openai
+    model_name: gpt-3.5-turbo
+    probe_spec: all
+    detector_spec: auto
+    extended_detectors: false
+    buff_spec:
+    buffs_include_original_prompt: false
+    buff_max:
+    detectors:
+      PythonPypi:
+        config:
+            cutoff_date: "1994-01-01 00:00:00 +0000"
+    generators: {}
+    buffs: {}
+    harnesses: {}
+    probe_spec: packagehallucination.Python
+    probes:
+        encoding:
+            payloads:
+                - default
+
+reporting:
+    report_prefix:
+    taxonomy:
+    report_dir: garak_runs
+    show_100_pass_modules: true

garak/detectors/packagehallucination.py

@@ -18,6 +18,7 @@
 Existing packages are also checked from the current version of Python's
 stdlib according to the stdlibs package."""
 
+from datetime import datetime
 import logging
 import re
 from typing import List, Set
@@ -47,7 +48,23 @@ def _load_package_list(self):
             f"Loading {self.language_name} package list from Hugging Face: {self.dataset_name}"
         )
         dataset = datasets.load_dataset(self.dataset_name, split="train")
-        self.packages = set(dataset["text"]) | set(stdlibs.module_names)
+
+        cutoff_date = _config.plugins.detectors[self.__class__.__name__]["config"]["cutoff_date"]
+        # Filter packages based on cutoff date if given
+        if cutoff_date:
+            try:
+                cutoff = datetime.fromisoformat(cutoff_date)
+                filtered_packages = [
+                    pkg for pkg, date_str in zip(dataset["text"], dataset["package_first_seen"])
+                    if datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S %z") <= cutoff
+                ]
+                print(len(filtered_packages))
+                self.packages = set(filtered_packages) | set(stdlibs.module_names)
+            except ValueError as e:
+                logging.warning(f"Invalid cutoff date format: {e}. Using all packages.")
+                self.packages = set(dataset["text"]) | set(stdlibs.module_names)
+        else:
+            self.packages = set(dataset["text"]) | set(stdlibs.module_names)
 
     def _extract_package_references(self, output: str) -> Set[str]:
         raise NotImplementedError
@@ -94,7 +111,7 @@ class PythonPypi(PackageHallucinationDetector):
     """Check if the output tries to import a package not listed in stdlib or a pypi archive listing"""
 
     DEFAULT_PARAMS = PackageHallucinationDetector.DEFAULT_PARAMS | {
-        "dataset_name": "garak-llm/pypi-20230724",
+        "dataset_name": "garak-llm/pypi-20241031",
         "language_name": "python",
     }
 
@@ -108,7 +125,7 @@ class RubyGems(PackageHallucinationDetector):
     """Check if the output tries to require a gem not listed in the Ruby standard library or RubyGems"""
 
     DEFAULT_PARAMS = PackageHallucinationDetector.DEFAULT_PARAMS | {
-        "dataset_name": "garak-llm/rubygems-20230301",
+        "dataset_name": "garak-llm/rubygems-20241031",
         "language_name": "ruby",
     }
 
@@ -126,7 +143,7 @@ class JavaScriptNpm(PackageHallucinationDetector):
     """Check if the output tries to import or require an npm package not listed in the npm registry"""
 
     DEFAULT_PARAMS = PackageHallucinationDetector.DEFAULT_PARAMS | {
-        "dataset_name": "garak-llm/npm-20240828",
+        "dataset_name": "garak-llm/npm-20241031",
         "language_name": "javascript",
     }
 

tools/packagehallucination/javascript/main.py

@@ -0,0 +1,102 @@
+import time
+import requests
+from datetime import datetime, timezone
+import backoff
+import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+TIME_FORMAT = "%Y-%m-%d %H:%M:%S %z"
+
+@backoff.on_exception(backoff.expo,
+                      (requests.exceptions.RequestException, requests.exceptions.HTTPError),
+                      max_tries=5)
+def get_package_first_seen(package_name):
+    url = f"https://registry.npmjs.org/{package_name}"
+    try:
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+        data = response.json()
+        created_date = data.get('time', {}).get('created', 'N/A')
+        # Parse the ISO format date and format it according to TIME_FORMAT
+        dt = datetime.fromisoformat(created_date)
+        dt = dt.replace(tzinfo=timezone.utc)
+        created_date = dt.strftime(TIME_FORMAT)
+    except requests.RequestException as e:
+        created_date = f"Error: {str(e)}"
+        print(f'Error getting data for {package_name}: {created_date}')
+
+    return created_date
+
+def main():
+    # names.json from https://github.com/nice-registry/all-the-package-names/blob/master/names.json
+    input_file = 'names.json'
+    output_file = 'npm_packages3.tsv'
+    processed = 0
+    included = 0
+    excluded = 0
+    errors = 0
+    start_time = time.time()
+
+    # Read the JSON file into a Python list
+    with open(input_file, 'r') as infile:
+        package_names = json.load(infile)
+
+    total_packages = len(package_names)
+    print(f"Starting to process {total_packages} npm packages...")
+
+    # Processes packages in parallel within batches
+    batch_size = 1000
+    batches = [package_names[i:i+batch_size] for i in range(0, len(package_names), batch_size)]
+
+    with open(output_file, 'a') as outfile:
+        outfile.write("text\tpackage_first_seen\n")
+        for batch in batches:
+            batch_results = []
+            with ThreadPoolExecutor(max_workers=batch_size) as executor:
+                future_to_package = {executor.submit(get_package_first_seen, package): package for package in batch}
+
+                for future in as_completed(future_to_package):
+                    package = future_to_package[future]
+                    creation_date = future.result()
+                    batch_results.append((package, creation_date))
+
+            batch_output = []
+            for package, creation_date in batch_results:
+                if creation_date:
+                    batch_output.append(f"{package}\t{creation_date}")
+                    included += 1
+                    status = "Included"
+                else:
+                    excluded += 1
+                    status = "Error" if "Error:" in str(creation_date) else "Excluded"
+
+                processed += 1
+
+                if "Error:" in str(creation_date):
+                    errors += 1
+
+            outfile.write("\n".join(batch_output) + "\n")
+            outfile.flush()
+
+            # Progress reporting
+            elapsed_time = time.time() - start_time
+            packages_per_second = processed / elapsed_time
+            estimated_total_time = total_packages / packages_per_second
+            estimated_remaining_time = estimated_total_time - elapsed_time
+
+            print(f"Processed: {processed}/{total_packages} ({processed/total_packages*100:.2f}%)")
+            print(f"Included: {included}, Excluded: {excluded}, Errors: {errors}")
+            print(f"Elapsed time: {elapsed_time:.2f} seconds")
+            print(f"Estimated remaining time: {estimated_remaining_time:.2f} seconds")
+            print(f"Processing speed: {packages_per_second:.2f} packages/second")
+            print("-" * 50)
+
+    print(f"Filtering complete. Results saved in {output_file}")
+    print(f"Total gems processed: {processed}")
+    print(f"Gems included: {included}")
+    print(f"Gems excluded: {excluded}")
+    print(f"Gems with errors: {errors}")
+    print(f"Total execution time: {time.time() - start_time:.2f} seconds")
+
+if __name__ == "__main__":
+    main()

tools/packagehallucination/python/main.py

@@ -0,0 +1,85 @@
+import requests
+from datetime import datetime, timezone
+import csv
+import backoff
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+TIME_FORMAT = "%Y-%m-%d %H:%M:%S %z"
+
+def get_all_packages():
+    url = "https://pypi.org/simple/"
+    response = requests.get(url)
+    packages = response.text.split("\n")
+    return [pkg.split("/")[2] for pkg in packages if "a href" in pkg]
+
+@backoff.on_exception(backoff.expo,
+                      (requests.exceptions.RequestException, requests.exceptions.HTTPError),
+                      max_tries=5)
+def get_package_first_seen(package_name):
+    url = f"https://pypi.org/pypi/{package_name}/json"
+    response = requests.get(url)
+    response.raise_for_status()
+    data = response.json()
+    releases = data.get("releases", {})
+    if releases:
+        oldest_release = min(releases.keys(), key=lambda x: releases[x][0]['upload_time'] if releases[x] else '9999-99-99')
+        if releases[oldest_release] and releases[oldest_release][0].get("upload_time"):
+            # Parse the upload time and format it according to TIME_FORMAT
+            upload_time = releases[oldest_release][0]["upload_time"]
+            try:
+                # Parse the time (PyPI times are in UTC)
+                dt = datetime.fromisoformat(upload_time)
+                dt = dt.replace(tzinfo=timezone.utc)
+                return dt.strftime(TIME_FORMAT)
+            except ValueError:
+                return None
+    return None
+
+def main():
+    output_file = "pypi_20241007_NEW.tsv"
+    packages = get_all_packages()
+    processed = 0
+    total_packages = len(packages)
+    print(f"Starting to process {total_packages} PyPI packages...")
+
+    batch_size = 1000
+    batches = [packages[i:i+batch_size] for i in range(0, total_packages, batch_size)]
+
+    try:
+        with open(output_file, "a", newline='') as outfile:
+            tsv_writer = csv.writer(outfile, delimiter='\t')
+            tsv_writer.writerow(["text", "package_first_seen"])
+
+            for batch in batches:
+                batch_results = []
+                with ThreadPoolExecutor(max_workers=batch_size) as executor:
+                    future_to_package = {executor.submit(get_package_first_seen, package): package for package in batch}
+
+                    for future in as_completed(future_to_package):
+                        package = future_to_package[future]
+                        try:
+                            creation_date = future.result()
+                            batch_results.append((package, creation_date))
+                            processed += 1
+                            if processed % 100 == 0:
+                                print(f"Processed: {processed}/{total_packages} ({processed/total_packages*100:.2f}%)")
+                        except Exception as e:
+                            print(f"Error processing {package}: {str(e)}")
+
+                for package, creation_date in batch_results:
+                    if creation_date:
+                        tsv_writer.writerow([package, creation_date])
+                    else:
+                        print(f"No creation date found for {package}")
+
+                outfile.flush()
+                print(f"Batch completed. Total processed: {processed}/{total_packages} ({processed/total_packages*100:.2f}%)")
+                print("*"*50)
+
+    except IOError as e:
+        print(f"Error writing to file: {str(e)}")
+
+    print(f"Done! Results saved in {output_file}")
+
+if __name__ == "__main__":
+    main()

tools/packagehallucination/ruby/main.py

@@ -0,0 +1,101 @@
+import time
+import requests
+from datetime import datetime, timezone
+import backoff
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+INPUT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+TIME_FORMAT = "%Y-%m-%d %H:%M:%S %z"
+
+@backoff.on_exception(backoff.expo,
+                      (requests.exceptions.RequestException, requests.exceptions.HTTPError),
+                      max_tries=5)
+def get_package_first_seen(gem_name):
+    url = f"https://rubygems.org/api/v1/versions/{gem_name}.json"
+    response = requests.get(url, timeout=30)
+    response.raise_for_status()  # This will raise an HTTPError for bad responses
+
+    versions = response.json()
+
+    # Sort versions by creation date and get the earliest one
+    earliest_version = min(versions, key=lambda v: datetime.strptime(v['created_at'], INPUT_TIME_FORMAT))
+
+    # Parse and format the date
+    creation_datetime = datetime.strptime(earliest_version['created_at'], INPUT_TIME_FORMAT)
+    creation_datetime = creation_datetime.replace(tzinfo=timezone.utc)
+    return creation_datetime.strftime(TIME_FORMAT)
+
+def main():
+    # gems.txt is the output from the `gem list --remote` command
+    input_file = 'gems.txt'
+    output_file = 'filtered_gems.tsv'
+    batch_size = 100
+
+    # Read all gem names first
+    with open(input_file, 'r') as infile:
+        all_gems = [line.strip().split(" (")[0] for line in infile]
+
+    total_gems = len(all_gems)
+    processed = 0
+    included = 0
+    excluded = 0
+    errors = 0
+    start_time = time.time()
+
+    # Create batches
+    batches = [all_gems[i:i+batch_size] for i in range(0, total_gems, batch_size)]
+
+    print(f"Starting to process {total_gems} gems...")
+
+    with open(output_file, 'a') as outfile:
+        outfile.write(f"text\tpackage_first_seen\n")
+
+        for batch in batches:
+            batch_results = []
+            with ThreadPoolExecutor(max_workers=batch_size) as executor:
+                future_to_gem = {executor.submit(get_package_first_seen, gem_name): gem_name for gem_name in batch}
+
+                for future in as_completed(future_to_gem):
+                    gem_name = future_to_gem[future]
+                    try:
+                        formatted_date = future.result()
+                        batch_results.append((gem_name, formatted_date))
+                        included += 1
+                        status = "Included"
+                    except Exception as e:
+                        print(f"Error processing gem '{gem_name}': {e}")
+                        errors += 1
+                        status = "Error"
+
+                    processed += 1
+
+                    if processed % 100 == 0 or processed == total_gems:
+                        elapsed_time = time.time() - start_time
+                        gems_per_second = processed / elapsed_time
+                        estimated_total_time = total_gems / gems_per_second
+                        estimated_remaining_time = estimated_total_time - elapsed_time
+
+                        print(f"Processed: {processed}/{total_gems} ({processed/total_gems*100:.2f}%)")
+                        print(f"Included: {included}, Excluded: {excluded}, Errors: {errors}")
+                        print(f"Elapsed time: {elapsed_time:.2f} seconds")
+                        print(f"Estimated remaining time: {estimated_remaining_time:.2f} seconds")
+                        print(f"Processing speed: {gems_per_second:.2f} gems/second")
+                        print("-" * 50)
+
+            # Write batch results
+            for gem_name, formatted_date in batch_results:
+                if formatted_date:
+                    outfile.write(f"{gem_name}\t{formatted_date}\n")
+            outfile.flush()
+            print(f"Batch completed. Total processed: {processed}/{total_gems} ({processed/total_gems*100:.2f}%)")
+            print("*"*50)
+
+    print(f"Filtering complete. Results saved in {output_file}")
+    print(f"Total gems processed: {processed}")
+    print(f"Gems included: {included}")
+    print(f"Gems excluded: {excluded}")
+    print(f"Gems with errors: {errors}")
+    print(f"Total execution time: {time.time() - start_time:.2f} seconds")
+
+if __name__ == "__main__":
+    main()