From 18e522bbefc3d162d0beeb02ea2c93f33b2b7159 Mon Sep 17 00:00:00 2001 From: Strongest Number 9 <16169054+StrongestNumber9@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:19:46 +0200 Subject: [PATCH] Initial release (#1) (#1) * Initial release --- .github/workflows/ci.yaml | 51 +++ .github/workflows/release.yaml | 60 ++++ .gitignore | 3 + README.adoc | 121 +++++-- docker/Dockerfile | 22 ++ docker/docker-entrypoint.sh | 57 ++++ pom.xml | 317 ++++++++++++++++++ rpm/pom.xml | 174 ++++++++++ rpm/src/main/resources/authorize.example.json | 8 + .../main/resources/credentials.example.json | 6 + rpm/src/main/resources/logback.xml | 16 + .../lookup.example/example_hosts.json | 11 + .../lookup.example/example_indexes.json | 11 + rpm/src/main/resources/pth_05.properties | 8 + rpm/src/main/resources/pth_05.service | 17 + src/main/assembly/jar-with-dependencies.xml | 80 +++++ .../java/com/teragrep/pth_05/ExitCode.java | 73 ++++ src/main/java/com/teragrep/pth_05/Main.java | 180 ++++++++++ .../pth_05/S3ProxyBlobStoreFactory.java | 110 ++++++ .../pth_05/TeragrepBlobStoreLocator.java | 116 +++++++ .../java/com/teragrep/pth_05/authz/Log.java | 114 +++++++ .../com/teragrep/pth_05/authz/Request.java | 71 ++++ .../pth_05/authz/RequestAuthorizer.java | 113 +++++++ .../pth_05/authz/RequestPathProcessor.java | 67 ++++ .../pth_05/authz/loggroup/LogGroup.java | 79 +++++ .../authz/loggroup/LogGroupProcessor.java | 132 ++++++++ .../pth_05/authz/loggroup/LogGroupSearch.java | 76 +++++ .../authz/loggroup/lookup/LookupTable.java | 89 +++++ .../loggroup/lookup/LookupTableEntry.java | 73 ++++ src/main/python/generate_index_mapping.py | 89 +++++ .../tests/TeragrepBlobStoreLocatorTest.java | 148 ++++++++ .../tests/authz/RequestAuthorizerTest.java | 80 +++++ .../tests/authz/RequestPathProcessorTest.java | 71 ++++ .../authz/loggroup/LogGroupProcessorTest.java | 77 +++++ .../authz/loggroup/LogGroupSearchTest.java | 75 +++++ src/test/resources/authorize.json | 26 ++ .../testtag/testtag.log-2021072608.log.gz | Bin 0 -> 37 bytes src/test/resources/credentials.json | 1 + src/test/resources/lookup/example_hosts.json | 15 + .../resources/lookup/example_indexes.json | 15 + .../resources/lookup/test-data_hosts.json | 15 + .../resources/lookup/test-data_indexes.json | 15 + src/test/resources/s3cfg.valid.conf | 83 +++++ src/test/resources/test.properties | 6 + 44 files changed, 2944 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .gitignore create mode 100644 docker/Dockerfile create mode 100755 docker/docker-entrypoint.sh create mode 100644 pom.xml create mode 100644 rpm/pom.xml create mode 100644 rpm/src/main/resources/authorize.example.json create mode 100644 rpm/src/main/resources/credentials.example.json create mode 100644 rpm/src/main/resources/logback.xml create mode 100644 rpm/src/main/resources/lookup.example/example_hosts.json create mode 100644 rpm/src/main/resources/lookup.example/example_indexes.json create mode 100644 rpm/src/main/resources/pth_05.properties create mode 100644 rpm/src/main/resources/pth_05.service create mode 100644 src/main/assembly/jar-with-dependencies.xml create mode 100644 src/main/java/com/teragrep/pth_05/ExitCode.java create mode 100644 src/main/java/com/teragrep/pth_05/Main.java create mode 100644 src/main/java/com/teragrep/pth_05/S3ProxyBlobStoreFactory.java create mode 100644 src/main/java/com/teragrep/pth_05/TeragrepBlobStoreLocator.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/Log.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/Request.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/RequestAuthorizer.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/RequestPathProcessor.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroup.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupProcessor.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupSearch.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTable.java create mode 100644 src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTableEntry.java create mode 100755 src/main/python/generate_index_mapping.py create mode 100644 src/test/java/com/teragrep/pth_05/tests/TeragrepBlobStoreLocatorTest.java create mode 100644 src/test/java/com/teragrep/pth_05/tests/authz/RequestAuthorizerTest.java create mode 100644 src/test/java/com/teragrep/pth_05/tests/authz/RequestPathProcessorTest.java create mode 100644 src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupProcessorTest.java create mode 100644 src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupSearchTest.java create mode 100644 src/test/resources/authorize.json create mode 100644 src/test/resources/blobstore/hundred-year/2021/07-26/testhost/testtag/testtag.log-2021072608.log.gz create mode 100644 src/test/resources/credentials.json create mode 100644 src/test/resources/lookup/example_hosts.json create mode 100644 src/test/resources/lookup/example_indexes.json create mode 100644 src/test/resources/lookup/test-data_hosts.json create mode 100644 src/test/resources/lookup/test-data_indexes.json create mode 100644 src/test/resources/s3cfg.valid.conf create mode 100644 src/test/resources/test.properties diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..fb09450 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,51 @@ +name: CI + +on: push + +jobs: + verify: + name: Verify Code + runs-on: ubuntu-latest + + env: + COVERITY: coverity_tool + + steps: + - uses: actions/checkout@v4 + + - name: Setup Maven Central + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'adopt' + + - name: Cache Local Maven Repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + + - name: Compile Test and Verify + run: mvn --batch-mode clean verify + + - name: Cache Coverity + id: cache_coverity + uses: actions/cache@v2 + with: + path: ${{ env.COVERITY }} + key: coverity + + - name: Download Coverity + if: steps.cache_coverity.outputs.cache-hit != 'true' + run: | + wget --quiet https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.COVERITY_TOKEN }}&project=pth_05" -O ${{ env.COVERITY }}.tgz + mkdir -p ${{ env.COVERITY }} + tar zxvf ${{ env.COVERITY }}.tgz -C ${{ env.COVERITY }} --strip-components 1 + + - name: Compile Coverity + run: | + ${{ env.COVERITY }}/bin/cov-build --dir cov-int mvn -DskipTests=true --batch-mode clean compile + tar czvf pth_05.tgz cov-int + + - name: Upload to Coverity + run: curl --silent --form token=${{ secrets.COVERITY_TOKEN }} --form email=${{ secrets.COVERITY_EMAIL }} --form file=@pth_05.tgz --form version="${GITHUB_REF##*/}" --form description="automated upload" https://scan.coverity.com/builds?project=pth_05 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..b94db82 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,60 @@ +name: Publish + +on: + release: + types: [published] + +jobs: + upload: + name: Upload + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + server-id: github + settings-path: ${{ github.workspace }} + + - name: Build jar + run: mvn --batch-mode -Drevision=${{ github.event.release.tag_name }} -Dsha1= -Dchangelist= clean package + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Build rpm + run: cd rpm/ && mvn --batch-mode -Drevision=${{ github.event.release.tag_name }} -Dsha1= -Dchangelist= -f rpm.pom.xml package + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Attach jar and rpm to release + uses: softprops/action-gh-release@v1 + with: + files: | + rpm/target/rpm/com.teragrep-pth_05/RPMS/noarch/com.teragrep-pth_05-*.noarch.rpm + target/pth_05-jar-with-dependencies.jar + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.5.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Lowercase repository name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + + - name: 'Build Image' + run: | + docker buildx build --output type=docker --tag ghcr.io/${{ env.REPO_LC }}:${{ github.event.release.tag_name }} --tag ghcr.io/${{ env.REPO_LC }}:latest . + docker push ghcr.io/${{ env.REPO_LC }} --all-tags diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6244e23 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target/ +.idea +rpm/target/ diff --git a/README.adoc b/README.adoc index fbd6ba3..2b619c4 100644 --- a/README.adoc +++ b/README.adoc @@ -1,47 +1,114 @@ -// Before publishing your new repository: -// 1. Write the readme file -// 2. Update the issues link in Contributing section in the readme file -// 3. Update the discussion link in config.yml file in .github/ISSUE_TEMPLATE directory += pth_05 -# repo-template +== Functionality -// Add a short description of your project. Tell what your project does and what it's used for. +pth_05 receives requests for OBJ-01 access via S3 protocol. +Requests are authenticated using credentials.json gathered by tool provided in the HDP-01 package. +Requested object path is then processed to determine host and tag parts of the object. +Host and tag is used to resolve index by searching the CFE-12 lookup files. +Request proceeds to authorization stage if index is found from the lookups. +Requests are authorized using authorize.json (generated by pth_05 provided generate_index_mapping.py tool) which contains index to unix group mappings. +Intersection is calculated by using the two sets of groups, one which are member of the index and the other which the identity is member of. +Request is accepted if any groups intersect within the two sets. -This is a template repository for Teragrep organization. +== Usage +pth_05 is a micro-service which can be deployed via RPM or a container. Only RPM +instructions are provided currently. However the Dockerfile shows what is to be +configured in order to get the container up and running as well. -## Features +=== Installation +pth_05 can be installed via the rpm package as follows: +[source,bash] +---- +yum install pth_05.rpm +---- -// List your project's features +=== Configuration -## Documentation +==== Data files +pth_05 requires following data available: -See the official documentation on https://docs.teragrep.com[docs.teragrep.com]. +* lookup-files +* authorize.json +* credentials.json -## Limitations +Lookup-files are sourced from CFE-12. Authorize.json-file can be generated with +the provided generate_index_mapping.py which ingests CFE-04 related +authorize.conf format. Credentials.json-file is produced by HDP-01 version 1.2.0 +or greater. -// If your project has limitations, please list them. Otherwise remove this section. +All of the files can be hand crafted and examples are provided within the RPM. -## How to [compile/use/implement] +==== Execution -// add instructions how people can start to use your project +Properties file configures the execution of pth_05 and is located at path: +/opt/teragrep/pth_05/etc/pth_05.properties -## Contributing +[source,properties] +---- +pth_05.endpoint=http://127.0.0.1:8080 +pth_05.credentials.file=/opt/teragrep/pth_05/etc/credentials.json +pth_05.authorize.file=/opt/teragrep/pth_05/etc/authorize.json +pth_05.lookup.path=/opt/teragrep/pth_05/etc/lookup +jclouds.provider=s3 +jclouds.identity=xxxxxxxxxxxxxxxxxxxx +jclouds.credential=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +jclouds.endpoint=https://s3.domain.tld +---- +Information about the variables -// Change the repository name in the issues link to match with your project's name +pth_05 specific: -You can involve yourself with our project by https://github.com/teragrep/repo-template/issues/new/choose[opening an issue] or submitting a pull request. +* pth_05.endpoint configures the address which the pth_05 listens at. +* pth_05.credentials.file is a path to the credentials.json-file. +* pth_05.authorize.file is a path to the authorize.json-file. +* pth_05.lookup.path is a path to the lookup directory. -Contribution requirements: +JClouds common: -. *All changes must be accompanied by a new or changed test.* If you think testing is not required in your pull request, include a sufficient explanation as why you think so. -. Security checks must pass -. Pull requests must align with the principles and http://www.extremeprogramming.org/values.html[values] of extreme programming. -. Pull requests must follow the principles of Object Thinking and Elegant Objects (EO). +* jclouds.provider configures which type of a service is proxied to. Use "s3". +* jclouds.identity configures the identity to the proxied target. +* jclouds.credential configures the credential to the proxied target. +* jclouds.endpoint configures the location to the proxied target. -Read more in our https://github.com/teragrep/teragrep/blob/main/contributing.adoc[Contributing Guideline]. +==== Memory settings -### Contributor License Agreement +Run `systemctl edit --full pth_05` to edit the memory settings. Currently the default values are: -Contributors must sign https://github.com/teragrep/teragrep/blob/main/cla.adoc[Teragrep Contributor License Agreement] before a pull request is accepted to organization's repositories. +`-Xms512m` -You need to submit the CLA only once. After submitting the CLA you can contribute to all Teragrep's repositories. +`-Xmx1024m` + +==== JMX + +Run `systemctl edit --full pth_05` and add the following to enable JMX: + +[systemctl edit --full pth_05] +---- + -Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=9996 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ +---- + +==== Logging + +pth_05 uses Logback to configure logging. Log configuration is at path: +/opt/teragrep/pth_05/etc/logback.xml + +=== Running +pth_05 is shipped with systemd service descriptor file and therefore is +available as a systemd-unit. +[source,bash] +---- +systemctl enable pth_05.service +systemctl start pth_05.service +---- + +=== Adding extra jars to classpath +Simply drop extra jars that might be required by logback configuration to `/opt/teragrep/pth_05/share/` + +== Build Artifacts +JAR +RPM +Container diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8f1c398 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,22 @@ +FROM centos:7 + +RUN yum -y install java-11-openjdk \ + && yum clean all + +COPY docker/target/pth_05.rpm / +RUN yum install -y /pth_05.rpm + +ENV PTH_05_ENDPOINT="http://0.0.0.0:8080" +ENV PTH_05_CREDENTIALS_FILE="/opt/teragrep/pth_05/etc/credentials.json" +ENV PTH_05_AUTHORIZE_FILE="/opt/teragrep/pth_05/etc/authorize.json" +ENV PTH_05_LOOKUP_PATH="/opt/teragrep/pth_05/etc/lookup" + +ENV PTH_05_PROVIDER="s3" +ENV PTH_05_PROVIDER_IDENTITY="default-identity" +ENV PTH_05_PROVIDER_CREDENTIAL="default-credential" +ENV PTH_05_PROVIDER_ENDPOINT="https://s3.default.tld:443" + +COPY docker/docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh + +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100755 index 0000000..77bb055 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# S3 Authorization enabled object gateway service pth_05 +# Copyright (C) 2021 Suomen Kanuuna Oy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# +# Additional permission under GNU Affero General Public License version 3 +# section 7 +# +# If you modify this Program, or any covered work, by linking or combining it +# with other code, such other code is not for that reason alone subject to any +# of the requirements of the GNU Affero GPL version 3 as long as this Program +# is the same Program as licensed from Suomen Kanuuna Oy without any additional +# modifications. +# +# Supplemented terms under GNU Affero General Public License version 3 +# section 7 +# +# Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified +# versions must be marked as "Modified version of" The Program. +# +# Names of the licensors and authors may not be used for publicity purposes. +# +# No rights are granted for use of trade names, trademarks, or service marks +# which are in The Program if any. +# +# Licensee must indemnify licensors and authors for any liability that these +# contractual assumptions impose on licensors and authors. +# +# To the extent this program is licensed as part of the Commercial versions of +# Teragrep, the applicable Commercial License may apply to this file if you as +# a licensee so wish it. + +java $JAVA_OPTIONS \ + -Dlogback.configurationFile=/opt/teragrep/pth_05/etc/logback.xml \ + -Dpth_05.endpoint=${PTH_05_ENDPOINT} \ + -Dpth_05.credentials.file=${PTH_05_CREDENTIALS_FILE} \ + -Dpth_05.authorize.file=${PTH_05_AUTHORIZE_FILE} \ + -Dpth_05.lookup.path=${PTH_05_LOOKUP_PATH} \ + -Djclouds.provider=${PTH_05_PROVIDER} \ + -Djclouds.identity=${PTH_05_PROVIDER_IDENTITY} \ + -Djclouds.credential=${PTH_05_PROVIDER_CREDENTIAL} \ + -Djclouds.endpoint=${PTH_05_PROVIDER_ENDPOINT} \ + -jar /opt/teragrep/pth_05/share/pth_05-jar-with-dependencies.jar diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..abf459d --- /dev/null +++ b/pom.xml @@ -0,0 +1,317 @@ + + 4.0.0 + com.teragrep + pth_05 + ${revision}${sha1}${changelist} + jar + pth_05 + + UTF-8 + 11 + ${java.version} + ${java.version} + ${java.version} + 2.3.0 + 1.7.28 + ${project.groupId}.shaded + 0.0.1 + -SNAPSHOT + + 0.4.3 + 1.1.0 + 0.2.0 + + + + + com.teragrep + jue_01 + ${teragrep.jue_01.version} + + + + com.teragrep + jai_01 + ${teragrep.jai_01.version} + + + + com.teragrep + jai_02 + ${teragrep.jai_02.version} + + + ch.qos.logback + logback-classic + 1.2.3 + + + javax.xml.bind + jaxb-api + 2.3.1 + + + org.apache.jclouds + jclouds-allblobstore + ${jclouds.version} + + + org.apache.jclouds.api + filesystem + ${jclouds.version} + + + org.apache.jclouds.driver + jclouds-slf4j + ${jclouds.version} + + + org.eclipse.jetty + jetty-servlet + 11.0.17 + + + org.slf4j + slf4j-api + ${slf4j.version} + + + javax.annotation + javax.annotation-api + 1.3.2 + + + org.gaul + s3proxy + 2.1.0 + + + com.google.code.gson + gson + 2.8.7 + + + org.junit.jupiter + junit-jupiter-engine + 5.4.0-RC1 + test + + + org.junit.platform + junit-platform-launcher + 1.4.0-RC1 + test + + + org.junit.jupiter + junit-jupiter-api + 5.4.0-RC1 + test + + + + ${artifactId} + + + org.apache.rat + apache-rat-plugin + 0.16.1 + false + + + test + + check + + + + + false + false + + + Teragrep + Teragrep Affero General Public License v3 + + + + + + + Suomen Kanuuna Oy + 2021 + + S3 Authorization enabled object gateway service pth_05 + + Teragrep + + + true + false + + + .git/** + .gitattributes + .gitignore + .gitmodules + + .github/workflows/* + .github/ISSUE_TEMPLATE/* + toolchains.xml + settings.xml + + rpm/rpm.pom.xml + pom.xml + + rpm/pom.xml + rpm/src/main/resources/* + + docker/Dockerfile + + src/test/resources/test.properties + src/test/resources/s3cfg.valid.conf + + README.adoc + + src/main/java/com/teragrep/pth_05/S3ProxyBlobStoreFactory.java + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.4.1 + + + enforce + none + + + enforce-maven + + enforce + + + + + 3.2.5 + + + + + + + + + 11 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + all + true + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.12.1 + + true + -Xlint:all + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.release} + ${maven.compiler.source} + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + true + true + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.1 + + + package + + shade + + + + + com.teragrep.pth_05.Main + + + false + + + org.eclipse.jetty:* + + META-INF/MANIFEST.MF + META-INF/LICENSE + META-INF/NOTICE.txt + about.html + + + + + + org.eclipse.jetty:* + + + + + org.eclipse.jetty + ${shade.prefix}.org.eclipse.jetty + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + src/main/assembly/jar-with-dependencies.xml + + + + com.teragrep.pth_05.Main + true + + + + + + make-assembly + package + + single + + + + + + + diff --git a/rpm/pom.xml b/rpm/pom.xml new file mode 100644 index 0000000..5d9d6d7 --- /dev/null +++ b/rpm/pom.xml @@ -0,0 +1,174 @@ + + + rpm + 4.0.0 + pth_05 + ${revision}${sha1}${changelist} + pth_05 + pth_05 + com.teragrep + + UTF-8 + 0.0.1 + -SNAPSHOT + + + + ${project.basedir}/target + + + maven-enforcer-plugin + 3.4.1 + + + enforce + none + + + enforce-maven + + enforce + + + + + 3.2.5 + + + + + + + + org.codehaus.mojo + rpm-maven-plugin + 2.2.0 + true + + + default-rpm + + rpm + + package + + + + ${project.groupId}-${project.artifactId} + ${project.groupId}-${project.artifactId} + ${project.version} + ${env.BUILD_ID} + Proprietary + Teragrep pth_05 + https://teragrep.com/ + Teragrep <servicedesk@teragrep.com> + Teragrep/pth_05 + ${project.groupId}-${project.artifactId} + true + srv-s3gw + srv-s3gw + 0644 + 0755 + + + /usr/lib/systemd/system + false + + + ${project.basedir}/src/main/resources/pth_05.service + + + + + /opt/teragrep/${project.artifactId}/etc + noreplace + true + 600 + 700 + srv-s3gw + srv-s3gw + true + + + ${project.basedir}/src/main/resources/pth_05.properties + + + ${project.basedir}/src/main/resources/authorize.example.json + + + ${project.basedir}/src/main/resources/credentials.example.json + + + ${project.basedir}/src/main/resources/logback.xml + + + + + /opt/teragrep/${project.artifactId}/etc/lookup.example + noreplace + true + 600 + 700 + srv-s3gw + srv-s3gw + true + + + ${project.basedir}/src/main/resources/lookup.example/example_hosts.json + + + ${project.basedir}/src/main/resources/lookup.example/example_indexes.json + + + + + /opt/teragrep/${project.artifactId}/share + true + 755 + 755 + srv-s3gw + srv-s3gw + true + + + ${project.basedir}/../target/pth_05-jar-with-dependencies.jar + + + + + /opt/teragrep/${project.artifactId}/bin + true + 755 + 755 + srv-s3gw + srv-s3gw + true + + + ${project.basedir}/../src/main/python/generate_index_mapping.py + + + + + + java-11-openjdk + + + + + + + + + diff --git a/rpm/src/main/resources/authorize.example.json b/rpm/src/main/resources/authorize.example.json new file mode 100644 index 0000000..2121799 --- /dev/null +++ b/rpm/src/main/resources/authorize.example.json @@ -0,0 +1,8 @@ +[ + { + "group": "root", + "allowedIndexes": [ + "example_index" + ] + } +] diff --git a/rpm/src/main/resources/credentials.example.json b/rpm/src/main/resources/credentials.example.json new file mode 100644 index 0000000..edbabc1 --- /dev/null +++ b/rpm/src/main/resources/credentials.example.json @@ -0,0 +1,6 @@ +[ + { + "identity": ":invalid:example:name:", + "credential": "p@ssword_4_3xample_ind3x" + } +] diff --git a/rpm/src/main/resources/logback.xml b/rpm/src/main/resources/logback.xml new file mode 100644 index 0000000..1f603a6 --- /dev/null +++ b/rpm/src/main/resources/logback.xml @@ -0,0 +1,16 @@ + + + + [s3proxy] %.-1p %d{MM-dd HH:mm:ss.SSS} %t %c{30}:%L %X{clientId}|%X{sessionId}:%X{messageId}:%X{fileId}] %m%n + + + ${LOG_LEVEL:-info} + + + + + + + + + diff --git a/rpm/src/main/resources/lookup.example/example_hosts.json b/rpm/src/main/resources/lookup.example/example_hosts.json new file mode 100644 index 0000000..46e77bc --- /dev/null +++ b/rpm/src/main/resources/lookup.example/example_hosts.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "nomatch": "unknown", + "type": "string", + "table": [ + { + "index": "host.example.domain.tld", + "value": true + } + ] +} diff --git a/rpm/src/main/resources/lookup.example/example_indexes.json b/rpm/src/main/resources/lookup.example/example_indexes.json new file mode 100644 index 0000000..6ac0953 --- /dev/null +++ b/rpm/src/main/resources/lookup.example/example_indexes.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "nomatch": "unknown", + "type": "string", + "table": [ + { + "index": "example_tag", + "value": "example_index" + } + ] +} diff --git a/rpm/src/main/resources/pth_05.properties b/rpm/src/main/resources/pth_05.properties new file mode 100644 index 0000000..e6c0299 --- /dev/null +++ b/rpm/src/main/resources/pth_05.properties @@ -0,0 +1,8 @@ +pth_05.endpoint=http://127.0.0.1:8080 +pth_05.credentials.file=/opt/teragrep/pth_05/etc/credentials.json +pth_05.authorize.file=/opt/teragrep/pth_05/etc/authorize.json +pth_05.lookup.path=/opt/teragrep/pth_05/etc/lookup +jclouds.provider=s3 +jclouds.identity=xxxxxxxxxxxxxxxxxxxx +jclouds.credential=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +jclouds.endpoint=https://s3.domain.tld diff --git a/rpm/src/main/resources/pth_05.service b/rpm/src/main/resources/pth_05.service new file mode 100644 index 0000000..af6f911 --- /dev/null +++ b/rpm/src/main/resources/pth_05.service @@ -0,0 +1,17 @@ +[Unit] +Description=com.teragrep.pth_05 +ConditionPathExists=/opt/teragrep/pth_05/share/pth_05-jar-with-dependencies.jar + +[Service] +ExecStart=/usr/lib/jvm/jre-11-openjdk/bin/java \ + -Xms512m \ + -Xmx1024m \ + -Dpth_05.propertiesFile=/opt/teragrep/pth_05/etc/pth_05.properties \ + -Dlogback.configurationFile=/opt/teragrep/pth_05/etc/logback.xml \ + -cp "/opt/teragrep/pth_05/share/*" \ + com.teragrep.pth_05.Main +User=srv-s3gw +WorkingDirectory=/opt/teragrep/pth_05/etc + +[Install] +WantedBy=multi-user.target diff --git a/src/main/assembly/jar-with-dependencies.xml b/src/main/assembly/jar-with-dependencies.xml new file mode 100644 index 0000000..0de6339 --- /dev/null +++ b/src/main/assembly/jar-with-dependencies.xml @@ -0,0 +1,80 @@ + + + jar-with-dependencies + + jar + + false + + + metaInf-services + + + + + + org.eclipse.jetty:* + + / + true + true + runtime + + + + + ${project.basedir}/src/main/config + / + + logback.xml + + true + + + diff --git a/src/main/java/com/teragrep/pth_05/ExitCode.java b/src/main/java/com/teragrep/pth_05/ExitCode.java new file mode 100644 index 0000000..adae9df --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/ExitCode.java @@ -0,0 +1,73 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05; + +public enum ExitCode { + OK(0), + BLOBSTORE_CREATION_ERROR(1), + BLOBSTORE_LOCATOR_CREATION_ERROR(2), + S3PROXY_START_ERROR(3), + // variable errors 200+ + PROPERTY_PTH_05_PROPERTIES_NO_SUCH_FILE(200), + PROPERTY_ENDPOINT(201), + PROPERTY_CREDENTIALS_FILE(202), + PROPERTY_AUTHORIZE_FILE(203), + PROPERTY_LOOKUP_PATH(204), + // default + DEFAULT(255) + ; + + private final int code; + + ExitCode(int code) { + this.code = code; + } + + public int getExitCode() { + return code; + } + } diff --git a/src/main/java/com/teragrep/pth_05/Main.java b/src/main/java/com/teragrep/pth_05/Main.java new file mode 100644 index 0000000..52bb056 --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/Main.java @@ -0,0 +1,180 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.eclipse.jetty.util.component.AbstractLifeCycle; +import org.gaul.s3proxy.AuthenticationType; +import org.gaul.s3proxy.S3Proxy; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.concurrent.DynamicExecutors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +public class Main { + private static final Logger logger = LoggerFactory.getLogger( + Main.class); + + public static void main(String[] args) { + ExitCode exitCode; + Properties properties = readPropertiesFile(); + + ThreadFactory factory = new ThreadFactoryBuilder() + .setNameFormat("user thread %d") + .setThreadFactory(Executors.defaultThreadFactory()) + .build(); + + ExecutorService executorService = DynamicExecutors.newScalingThreadPool( + 1, 20, 60 * 1000, factory); + + + BlobStore blobStore = null; + try { + blobStore = S3ProxyBlobStoreFactory.createBlobStore( + properties, executorService + ); + } catch (IOException e) { + logger.error(e.toString()); + exitCode = ExitCode.BLOBSTORE_CREATION_ERROR; + System.exit(exitCode.getExitCode()); + } + + + S3Proxy s3Proxy = S3Proxy.builder() + .awsAuthentication(AuthenticationType.AWS_V2_OR_V4, "", "") + .blobStore(blobStore) + .endpoint(URI.create(properties.getProperty("pth_05.endpoint"))) + .build(); + + // our authorizing blobStoreLocator + TeragrepBlobStoreLocator bLocator = null; + try { + bLocator = new TeragrepBlobStoreLocator( + properties.getProperty("pth_05.credentials.file"), + properties.getProperty("pth_05.authorize.file"), + properties.getProperty("pth_05.lookup.path") + + ); + } catch (IOException e) { + logger.error(e.toString()); + exitCode = ExitCode.BLOBSTORE_LOCATOR_CREATION_ERROR; + System.exit(exitCode.getExitCode()); + } + bLocator.setBlobStore(blobStore); + s3Proxy.setBlobStoreLocator(bLocator); + + try { + s3Proxy.start(); + } catch (Exception e) { + logger.error(e.toString()); + exitCode = ExitCode.S3PROXY_START_ERROR; + System.exit(exitCode.getExitCode()); + } + while (!s3Proxy.getState().equals(AbstractLifeCycle.STARTED)) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + logger.warn(e.toString()); + } + } + } + + private static Properties readPropertiesFile() { + ExitCode exitCode; + Properties systemProperties = System.getProperties(); + final String pth05PropertiesFile = systemProperties.getProperty("pth_05.propertiesFile"); + if ( pth05PropertiesFile != null) { + try { + final FileReader reader = new FileReader(pth05PropertiesFile); + systemProperties.load(reader); + reader.close(); + } catch (IOException e) { + logger.error(e.toString()); + exitCode = ExitCode.PROPERTY_PTH_05_PROPERTIES_NO_SUCH_FILE; + System.exit(exitCode.getExitCode()); + } + } + + // require endpoint + if (systemProperties.getProperty("pth_05.endpoint") == null) { + exitCode = ExitCode.PROPERTY_ENDPOINT; + logger.error("pth_05.endpoint" + " not set, existing."); + System.exit(exitCode.getExitCode()); + } + + // require credentials.file + if (systemProperties.getProperty("pth_05.credentials.file") == null) { + exitCode = ExitCode.PROPERTY_CREDENTIALS_FILE; + logger.error("pth_05.credentials.file" + " not set, existing."); + System.exit(exitCode.getExitCode()); + } + + // require authorize.file + if (systemProperties.getProperty("pth_05.authorize.file") == null) { + exitCode = ExitCode.PROPERTY_AUTHORIZE_FILE; + logger.error("pth_05.authorize.file" + " not set, existing."); + System.exit(exitCode.getExitCode()); + } + + // require lookup.path + if (systemProperties.getProperty("pth_05.lookup.path") == null) { + exitCode = ExitCode.PROPERTY_LOOKUP_PATH; + logger.error("pth_05.lookup.path" + " not set, existing."); + System.exit(exitCode.getExitCode()); + } + + return systemProperties; + } +} diff --git a/src/main/java/com/teragrep/pth_05/S3ProxyBlobStoreFactory.java b/src/main/java/com/teragrep/pth_05/S3ProxyBlobStoreFactory.java new file mode 100644 index 0000000..feb56b7 --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/S3ProxyBlobStoreFactory.java @@ -0,0 +1,110 @@ +package com.teragrep.pth_05; + +// From: org.gaul.s3proxy.Main, was private, now public + +/* + * Copyright 2014-2020 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import com.google.inject.Module; +import org.jclouds.Constants; +import org.jclouds.ContextBuilder; +import org.jclouds.JcloudsVersion; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.concurrent.config.ExecutorServiceModule; +import org.jclouds.location.reference.LocationConstants; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Properties; +import java.util.concurrent.ExecutorService; + + +public class S3ProxyBlobStoreFactory { + public static BlobStore createBlobStore(Properties properties, + ExecutorService executorService) throws IOException { + String provider = properties.getProperty(Constants.PROPERTY_PROVIDER); + String identity = properties.getProperty(Constants.PROPERTY_IDENTITY); + String credential = properties.getProperty( + Constants.PROPERTY_CREDENTIAL); + String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT); + properties.remove(Constants.PROPERTY_ENDPOINT); + String region = properties.getProperty( + LocationConstants.PROPERTY_REGION); + + if (provider == null) { + System.err.println( + "Properties file must contain: " + + Constants.PROPERTY_PROVIDER); + System.exit(1); + } + + if (provider.equals("filesystem") || provider.equals("transient")) { + // local blobstores do not require credentials + identity = Strings.nullToEmpty(identity); + credential = Strings.nullToEmpty(credential); + } else if (provider.equals("google-cloud-storage")) { + File credentialFile = new File(credential); + if (credentialFile.exists()) { + credential = Files.asCharSource(credentialFile, + StandardCharsets.UTF_8).read(); + } + properties.remove(Constants.PROPERTY_CREDENTIAL); + } + + if (identity == null || credential == null) { + System.err.println( + "Properties file must contain: " + + Constants.PROPERTY_IDENTITY + " and " + + Constants.PROPERTY_CREDENTIAL); + System.exit(1); + } + + properties.setProperty(Constants.PROPERTY_USER_AGENT, + String.format("s3proxy/%s jclouds/%s java/%s", + Main.class.getPackage().getImplementationVersion(), + JcloudsVersion.get(), + System.getProperty("java.version"))); + + ContextBuilder builder = ContextBuilder + .newBuilder(provider) + .credentials(identity, credential) + .modules(ImmutableList.of( + new SLF4JLoggingModule(), + new ExecutorServiceModule(executorService))) + .overrides(properties); + if (!Strings.isNullOrEmpty(endpoint)) { + builder = builder.endpoint(endpoint); + } + + BlobStoreContext context = builder.build(BlobStoreContext.class); + BlobStore blobStore; + if (context instanceof RegionScopedBlobStoreContext && + region != null) { + blobStore = ((RegionScopedBlobStoreContext) context) + .getBlobStore(region); + } else { + blobStore = context.getBlobStore(); + } + return blobStore; + } +} diff --git a/src/main/java/com/teragrep/pth_05/TeragrepBlobStoreLocator.java b/src/main/java/com/teragrep/pth_05/TeragrepBlobStoreLocator.java new file mode 100644 index 0000000..c4cd353 --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/TeragrepBlobStoreLocator.java @@ -0,0 +1,116 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05; + +import com.google.common.collect.Maps; +import com.teragrep.jai_02.ICredentialLookup; +import com.teragrep.jai_02.ReloadingCredentialLookup; +import com.teragrep.pth_05.authz.RequestAuthorizer; +import org.gaul.s3proxy.BlobStoreLocator; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.rest.AuthorizationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +public class TeragrepBlobStoreLocator implements BlobStoreLocator { + + private static final Logger logger = LoggerFactory.getLogger( + TeragrepBlobStoreLocator.class); + + /** + * BlobStoreLocator is used to find relevant blobstore, + * and the EXPECTED AWS SIGNATURE for the current + * request. BlobStoreLocator gives out the credential + * during the process which then is used to construct + * the EXPECTED AWS SIGNATURE. Then the EXPECTED AWS SIGNATURE + * is compared with the one within the request, + * and only if they match the blobstore is used. + * The BlobStoreLocator uses the identity to find the + * relevant blobstore and the relevant credential for it. + */ + + BlobStore blobStore = null; + final ICredentialLookup credentialLookup; + final RequestAuthorizer requestAuthorizer; + + public TeragrepBlobStoreLocator(String credentialsJSON, String authorizeJSON, String lookupPath) throws IOException { + this.credentialLookup = new ReloadingCredentialLookup(credentialsJSON, 300); + requestAuthorizer = new RequestAuthorizer(authorizeJSON, lookupPath); + } + + + public Map.Entry locateBlobStore(String identity, + String container, + String blob) { + /* + identity: bogus-identity + container: foobucket + blob: asd/das/zoom.jpg + */ + Map.Entry cred2blobstore = null; + + if (identity != null && blob != null) { + final String credential = credentialLookup.getCredential(identity); + if (credential != null) { + try { + requestAuthorizer.authorize(identity, container, blob); + cred2blobstore = Maps.immutableEntry(credential, blobStore); + } catch (IOException | AuthorizationException e) { + logger.error(e.toString()); + } + } + } + return cred2blobstore; + } + + public void setBlobStore(BlobStore blobStore) { + this.blobStore = blobStore; + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/Log.java b/src/main/java/com/teragrep/pth_05/authz/Log.java new file mode 100644 index 0000000..efe2797 --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/Log.java @@ -0,0 +1,114 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz; + +import com.google.gson.JsonObject; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashSet; +import java.util.UUID; + +public class Log { + + public static String authorization( + final String identity, + final HashSet indexes, + final String container, + final String blob, + final Boolean success) { + String outcome; + + if (success) { + outcome = "OK"; + } + else { + outcome = "NOK"; + } + + // request type_info + JsonObject typeInfo = new JsonObject(); + typeInfo.addProperty("request_id", ""); + typeInfo.addProperty("session_id", ""); + typeInfo.addProperty("subject", identity); + typeInfo.addProperty("predicate", "GRANT"); + typeInfo.addProperty("object", indexes.toString()); + typeInfo.addProperty("outcome", outcome); + + // content + JsonObject content = new JsonObject(); + content.addProperty("container", container); + content.addProperty("blob", blob); + + // common info + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("timestamp", + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX") + .format(new Date())); + jsonObject.addProperty("version", "1"); + jsonObject.addProperty("application", "teragrep"); + jsonObject.addProperty("environment", ""); + jsonObject.addProperty("component", "pth_05"); + try { + jsonObject.addProperty("instance", InetAddress.getLocalHost().getHostName()); + } catch (UnknownHostException e) { + jsonObject.addProperty("instance",""); + } + jsonObject.addProperty("retention", ""); + jsonObject.addProperty("uuid", UUID.randomUUID().toString()); + jsonObject.addProperty("type", "authorization"); + jsonObject.add("type_info", typeInfo); + + jsonObject.add("content", content); + + + + return jsonObject.toString(); + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/Request.java b/src/main/java/com/teragrep/pth_05/authz/Request.java new file mode 100644 index 0000000..ac48cfb --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/Request.java @@ -0,0 +1,71 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz; + +public final class Request { + final String year; + final String monthDay; + final String host; + final String tag; + final String file; + + public Request(String year, String monthDay, String host, String tag, String file) { + this.year = year; + this.monthDay = monthDay; + this.host = host; + this.tag = tag; + this.file = file; + } + + public String getHost() { + return host; + } + + public String getTag() { + return tag; + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/RequestAuthorizer.java b/src/main/java/com/teragrep/pth_05/authz/RequestAuthorizer.java new file mode 100644 index 0000000..5bc4e1a --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/RequestAuthorizer.java @@ -0,0 +1,113 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz; + +import com.teragrep.jai_01.IAuthorizationInfoProcessor; +import com.teragrep.jai_01.ReloadingAuthorizationInfoProcessor; +import com.teragrep.jue_01.UnixGroupSearch; +import com.teragrep.pth_05.authz.loggroup.LogGroup; +import com.teragrep.pth_05.authz.loggroup.LogGroupProcessor; +import org.jclouds.rest.AuthorizationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashSet; +import java.util.LinkedList; + +public final class RequestAuthorizer { + + private static final Logger logger = LoggerFactory.getLogger( + RequestAuthorizer.class); + + final LinkedList logGroupList; + final IAuthorizationInfoProcessor authorizationInfoProcessor; + final UnixGroupSearch unixGroupSearch; + + public RequestAuthorizer(String authorizeJSON, String lookupPath) throws IOException { + final LogGroupProcessor logGroupProcessor = new LogGroupProcessor(); + this.logGroupList = logGroupProcessor.load(lookupPath); + + this.authorizationInfoProcessor = new ReloadingAuthorizationInfoProcessor(authorizeJSON, 300); + this.unixGroupSearch = new UnixGroupSearch(); + } + + private HashSet getIndexes(String host, String tag) { + final HashSet indexes = new HashSet<>(); + // CFE-12 lookups have no way to determine all groups for host, iterating all + for (LogGroup group : logGroupList) { + final String index = group.getIndex(host,tag); + if (index != null) { + indexes.add(index); + } + } + return indexes; + } + + public void authorize(String identity, String container, String blob) throws IOException { + + final Request request = RequestPathProcessor.process(identity, blob); + final HashSet indexes = getIndexes(request.getHost(),request.getTag()); + final HashSet indexesGroupSet = authorizationInfoProcessor.getGroupSetForIndexes(indexes); + final HashSet origIdentityMemberOfSet = unixGroupSearch.getGroups(identity); + final HashSet identityMemberOfSet = new HashSet<>(origIdentityMemberOfSet); + + identityMemberOfSet.retainAll(indexesGroupSet); + if(identityMemberOfSet.size() == 0) { + logger.info(Log.authorization(identity, indexes, container, blob, false)); + throw new AuthorizationException("Access to: [" + blob + + "] denied for " + "identity [" + identity + + "] who is member of groups <" + + origIdentityMemberOfSet + + "> but allowed groups for indexes <[" + indexes + + "]> are: <" + indexesGroupSet + ">"); + } + else { + logger.info(Log.authorization(identity, indexes, container, blob, true)); + } + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/RequestPathProcessor.java b/src/main/java/com/teragrep/pth_05/authz/RequestPathProcessor.java new file mode 100644 index 0000000..0f8b228 --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/RequestPathProcessor.java @@ -0,0 +1,67 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz; + +import org.jclouds.rest.AuthorizationException; + +public final class RequestPathProcessor { + private RequestPathProcessor() { + + } + + public static Request process(String identity, String blob) { + final String[] path = blob.split("/", 5); + + if (path.length != 5) { + throw new AuthorizationException("Malformed request " + + "expecting: year/month-day/host/tag/file format. Got: " + + "[" + blob + "] by identity [" + identity + "]"); + } + + return new Request(path[0], path[1], path[2], path[3], path[4]); + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroup.java b/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroup.java new file mode 100644 index 0000000..65b272b --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroup.java @@ -0,0 +1,79 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz.loggroup; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Set; + +public final class LogGroup { + final String name; + final Set hosts; + final HashMap tag2index; + + public LogGroup(String name, Set hosts, HashMap tag2index) { + this.name = name; + this.hosts = hosts; + this.tag2index = tag2index; + } + + public String getIndex(String host, String tag) { + if (this.hosts.contains(host.toLowerCase(Locale.ROOT))) { + return this.tag2index.get(tag.toLowerCase(Locale.ROOT)); + } + return null; + } + + @Override + public String toString() { + return "LogGroup{" + + "name='" + name + '\'' + + ", hosts=" + hosts + + ", tag2index=" + tag2index + + '}'; + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupProcessor.java b/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupProcessor.java new file mode 100644 index 0000000..4d376cf --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupProcessor.java @@ -0,0 +1,132 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz.loggroup; + +import com.google.gson.Gson; +import com.teragrep.pth_05.authz.RequestAuthorizer; +import com.teragrep.pth_05.authz.loggroup.lookup.LookupTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +public final class LogGroupProcessor { + private static final Logger logger = LoggerFactory.getLogger( + LogGroupProcessor.class); + + final Gson gson; + + public LogGroupProcessor() { + this.gson = new Gson(); + } + + public LinkedList load(String path) throws IOException { + final LinkedList logGroupList = new LinkedList<>(); + Set groups; + + try (DirectoryStream stream = Files.newDirectoryStream(Paths.get(path))) { + LogGroupSearch groupSearch = new LogGroupSearch(stream); + groups = groupSearch.getGroupList(); + + } + + //logger.debug("Found log group names <[" + groups + "]>"); + + if (groups != null && groups.size() > 0) { + + for (String group : groups) { + // read hosts for the Group + final Set hosts = loadHostsFile(path, group); + + // read indexes for the Group + final HashMap tag2index = loadIndexesFile(path, group); + + logGroupList.add(new LogGroup(group, hosts, tag2index)); + } + } + + //logger.debug("Loaded log group list <[" + logGroupList + "]>"); + + return logGroupList; + } + + private Set loadHostsFile(String path, String group) throws FileNotFoundException { + BufferedReader hostReader = new BufferedReader( + new FileReader(path + "/" + group + "_hosts.json")); + LookupTable hostsTable = gson.fromJson(hostReader, LookupTable.class); + + final Set groupHosts = new HashSet<>(); + for (int i = 0; i < hostsTable.getTable().size(); i++) { + if ("true".equals(hostsTable.getTable().get(i).getValue())) + groupHosts.add(hostsTable.getTable().get(i).getIndex().toLowerCase(Locale.ROOT)); + } + return groupHosts; + } + + private HashMap loadIndexesFile(String path, String group) throws FileNotFoundException { + BufferedReader indexesReader = new BufferedReader( + new FileReader(path + "/" + group + "_indexes.json")); + LookupTable indexesTable = gson.fromJson(indexesReader, LookupTable.class); + + final HashMap groupIndexes = new HashMap<>(); + for (int i = 0; i < indexesTable.getTable().size(); i++) { + groupIndexes.put( + indexesTable.getTable().get(i).getIndex().toLowerCase(Locale.ROOT), // tag + indexesTable.getTable().get(i).getValue().toLowerCase(Locale.ROOT) // index + ); + } + return groupIndexes; + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupSearch.java b/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupSearch.java new file mode 100644 index 0000000..2af54cf --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/loggroup/LogGroupSearch.java @@ -0,0 +1,76 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz.loggroup; + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; + +public final class LogGroupSearch { + + final Set groupList; + + public LogGroupSearch(DirectoryStream stream) { + this.groupList = new HashSet<>(); + + for (Path path : stream) { + if (!Files.isDirectory(path)) { + groupList.add(path.getFileName() + .toString() + .replaceAll("_hosts\\.json$", "") + .replaceAll("_indexes\\.json$", "")); + } + } + } + + public Set getGroupList() { + return this.groupList; + } + +} diff --git a/src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTable.java b/src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTable.java new file mode 100644 index 0000000..6181deb --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTable.java @@ -0,0 +1,89 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz.loggroup.lookup; + +import java.util.List; + +public class LookupTable { + final int version; + final String nomatch; + final String type; + final List table; + + public LookupTable(int version, String nomatch, String type, List table) { + this.version = version; + this.nomatch = nomatch; + this.type = type; + this.table = table; + } + + public int getVersion() { + return version; + } + + public String getNomatch() { + return nomatch; + } + + public String getType() { + return type; + } + + public List getTable() { + return table; + } + + @Override + public String toString() { + return "LookupTable{" + + "version=" + version + + ", nomatch='" + nomatch + '\'' + + ", type='" + type + '\'' + + ", table=" + table + + '}'; + } +} diff --git a/src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTableEntry.java b/src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTableEntry.java new file mode 100644 index 0000000..55224a9 --- /dev/null +++ b/src/main/java/com/teragrep/pth_05/authz/loggroup/lookup/LookupTableEntry.java @@ -0,0 +1,73 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.authz.loggroup.lookup; + +public class LookupTableEntry { + final String index; + final String value; + + public LookupTableEntry(String index, String value) { + this.index = index; + this.value = value; + } + + public String getIndex() { + return index; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "LookupTableEntry{" + + "index='" + index + '\'' + + ", value='" + value + '\'' + + '}'; + } +} diff --git a/src/main/python/generate_index_mapping.py b/src/main/python/generate_index_mapping.py new file mode 100755 index 0000000..8de32a3 --- /dev/null +++ b/src/main/python/generate_index_mapping.py @@ -0,0 +1,89 @@ +#!/usr/bin/python3 + +# S3 Authorization enabled object gateway service pth_05 +# Copyright (C) 2021 Suomen Kanuuna Oy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# +# Additional permission under GNU Affero General Public License version 3 +# section 7 +# +# If you modify this Program, or any covered work, by linking or combining it +# with other code, such other code is not for that reason alone subject to any +# of the requirements of the GNU Affero GPL version 3 as long as this Program +# is the same Program as licensed from Suomen Kanuuna Oy without any additional +# modifications. +# +# Supplemented terms under GNU Affero General Public License version 3 +# section 7 +# +# Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified +# versions must be marked as "Modified version of" The Program. +# +# Names of the licensors and authors may not be used for publicity purposes. +# +# No rights are granted for use of trade names, trademarks, or service marks +# which are in The Program if any. +# +# Licensee must indemnify licensors and authors for any liability that these +# contractual assumptions impose on licensors and authors. +# +# To the extent this program is licensed as part of the Commercial versions of +# Teragrep, the applicable Commercial License may apply to this file if you as +# a licensee so wish it. + +import configparser +import sys +import os +import json + +# Usage +if len(sys.argv) < 2: + print(f"Usage: python3 {sys.argv[0]} path/to/config/dir") + sys.exit(0) + +# sanity checks +config_path = sys.argv[1] +if not os.path.isdir(config_path): + print(f"Failure: Argument given is not a directory or does not exist") + sys.exit(1) + +if not os.path.isfile(f"{config_path}/authentication.conf") or not os.path.isfile(f"{config_path}/authorize.conf"): + print(f"Can't find authentication.conf and/or authorization.conf from '{config_path}'") + sys.exit(1) + +# Read configs +authentication = configparser.ConfigParser(strict=False) +authentication.read(f"{config_path}/authentication.conf") + +authorize = configparser.ConfigParser(strict=False) +authorize.read(f"{config_path}/authorize.conf") + +# Container for authentications +groups = {} +for key in authentication: + # Ignore irrelevant + if key.startswith("roleMap"): + for usermap in authentication[key]: + groups[usermap] = authentication[key][usermap] + +# Find permissions for all groups +permission_list = [] +for key in authorize: + if key.removeprefix("role_") in groups and "srchIndexesAllowed" in authorize[key]: + if len(groups[key.removeprefix("role_")]) != 0 and len(authorize[key]["srchIndexesAllowed"]) != 0: + permission_list.append({"group": groups[key.removeprefix("role_")], "allowedIndexes": authorize[key]["srchIndexesAllowed"].split(";")}) + +print(json.dumps(permission_list)) diff --git a/src/test/java/com/teragrep/pth_05/tests/TeragrepBlobStoreLocatorTest.java b/src/test/java/com/teragrep/pth_05/tests/TeragrepBlobStoreLocatorTest.java new file mode 100644 index 0000000..d5382d0 --- /dev/null +++ b/src/test/java/com/teragrep/pth_05/tests/TeragrepBlobStoreLocatorTest.java @@ -0,0 +1,148 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.tests; + +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Module; +import com.teragrep.pth_05.TeragrepBlobStoreLocator; + +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.concurrent.DynamicExecutors; +import org.jclouds.concurrent.config.ExecutorServiceModule; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +public class TeragrepBlobStoreLocatorTest { + @Test + public void testLocateBlobStore() throws IOException { + TeragrepBlobStoreLocator blobStoreLocator = new TeragrepBlobStoreLocator("src/test/resources/credentials.json", "src/test/resources/authorize.json", "src/test/resources/lookup"); + + Properties properties = new Properties(); + properties.setProperty("jclouds.filesystem.basedir", "/tmp/testLocateBlobStore"); + + final String provider = "filesystem"; + + ThreadFactory factory = new ThreadFactoryBuilder() + .setNameFormat("test thread %d") + .setThreadFactory(Executors.defaultThreadFactory()) + .build(); + + ExecutorService executorService = DynamicExecutors.newScalingThreadPool( + 1, 20, 60 * 1000, factory); + + ContextBuilder builder = ContextBuilder + .newBuilder(provider) + .modules(ImmutableList.of( + new SLF4JLoggingModule(), + new ExecutorServiceModule(executorService))) + .overrides(properties); + + BlobStoreContext context = builder.build(BlobStoreContext.class); + + blobStoreLocator.setBlobStore(context.getBlobStore()); + + Map.Entry credentialToBlobStore = blobStoreLocator. + locateBlobStore( + "root" , + "100year-bucket", + "2021/07-22/testhost/testtag/testtag.log-2021072210.log.gz" + ); + + Assertions.assertEquals("aP051Xd3f1n3d@account", credentialToBlobStore.getKey()); + Assertions.assertNotNull(credentialToBlobStore.getValue()); + } + + + @Test + public void failLocateBlobStore() throws IOException { + TeragrepBlobStoreLocator blobStoreLocator = new TeragrepBlobStoreLocator("src/test/resources/credentials.json", "src/test/resources/authorize.json", "src/test/resources/lookup"); + + Properties properties = new Properties(); + properties.setProperty("jclouds.filesystem.basedir", "/tmp/testLocateBlobStore"); + + final String provider = "filesystem"; + + ThreadFactory factory = new ThreadFactoryBuilder() + .setNameFormat("test thread %d") + .setThreadFactory(Executors.defaultThreadFactory()) + .build(); + + ExecutorService executorService = DynamicExecutors.newScalingThreadPool( + 1, 20, 60 * 1000, factory); + + ContextBuilder builder = ContextBuilder + .newBuilder(provider) + .modules(ImmutableList.of( + new SLF4JLoggingModule(), + new ExecutorServiceModule(executorService))) + .overrides(properties); + + BlobStoreContext context = builder.build(BlobStoreContext.class); + + blobStoreLocator.setBlobStore(context.getBlobStore()); + + Map.Entry credentialToBlobStore = blobStoreLocator. + locateBlobStore( + "nobody" , + "5year-bucket", + "2021/07-22/testhost/testtag/testtag.log-2021072210.log.gz" + ); + + Assertions.assertNull(credentialToBlobStore); + } +} diff --git a/src/test/java/com/teragrep/pth_05/tests/authz/RequestAuthorizerTest.java b/src/test/java/com/teragrep/pth_05/tests/authz/RequestAuthorizerTest.java new file mode 100644 index 0000000..e54eabb --- /dev/null +++ b/src/test/java/com/teragrep/pth_05/tests/authz/RequestAuthorizerTest.java @@ -0,0 +1,80 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.tests.authz; + +import com.teragrep.pth_05.authz.RequestAuthorizer; +import org.jclouds.rest.AuthorizationException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +public class RequestAuthorizerTest { + @Test + public void authorizeRequestTest() throws IOException { + RequestAuthorizer requestAuthorizer = new RequestAuthorizer("src/test/resources/authorize.json", "src/test/resources/lookup"); + requestAuthorizer.authorize("root", "100-years", "2021/07-22/testhost/testtag/testtag.log-2021072210.log.gz"); + } + + @Test + public void rejectRequestTest() throws IOException { + RequestAuthorizer requestAuthorizer = new RequestAuthorizer("src/test/resources/authorize.json", "src/test/resources/lookup"); + Assertions.assertThrows( + AuthorizationException.class, + () -> requestAuthorizer.authorize("nobody", "100-years", "2021/07-22/testhost/testtag/testtag.log-2021072210.log.gz"), + "requestAuthorizer did not throw for a request without authorization"); + } + + @Test + public void lmsKubeFailure() throws IOException { + RequestAuthorizer requestAuthorizer = new RequestAuthorizer("src/test/resources/authorize.json", "src/test/resources/lookup"); + Assertions.assertThrows( + AuthorizationException.class, + () -> requestAuthorizer.authorize("root", "100-years", "2020/10-20/no-such-host/no-such-tag/no-such-tag-2020102023.log.gz"), + "requestAuthorizer did not throw for a request without authorization"); + } +} diff --git a/src/test/java/com/teragrep/pth_05/tests/authz/RequestPathProcessorTest.java b/src/test/java/com/teragrep/pth_05/tests/authz/RequestPathProcessorTest.java new file mode 100644 index 0000000..82e366a --- /dev/null +++ b/src/test/java/com/teragrep/pth_05/tests/authz/RequestPathProcessorTest.java @@ -0,0 +1,71 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.tests.authz; + +import com.teragrep.pth_05.authz.Request; +import com.teragrep.pth_05.authz.RequestPathProcessor; +import org.jclouds.rest.AuthorizationException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class RequestPathProcessorTest { + + @Test + public void testValidBlobPath() { + Request request = RequestPathProcessor.process("testValidBlobPath", "2021/07-22/testhost/testtag/testtag.log-2021072210.log.gz"); + Assertions.assertEquals("testhost", request.getHost()); + Assertions.assertEquals("testtag", request.getTag()); + } + + @Test + public void testMalformedBlobPath() { + Assertions.assertThrows( + AuthorizationException.class, + () -> RequestPathProcessor.process("testMalformedBlobPath", "2021/07-22/testhost/testtag.log-2021072210.log.gz"), + "requestPathProcessor did not throw on malformed path"); + } +} diff --git a/src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupProcessorTest.java b/src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupProcessorTest.java new file mode 100644 index 0000000..cbedb82 --- /dev/null +++ b/src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupProcessorTest.java @@ -0,0 +1,77 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.tests.authz.loggroup; + +import com.teragrep.pth_05.authz.loggroup.LogGroup; +import com.teragrep.pth_05.authz.loggroup.LogGroupProcessor; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.LinkedList; + +public class LogGroupProcessorTest { + + @Test + public void loadTest() throws IOException { + LogGroupProcessor logGroupProcessor = new LogGroupProcessor(); + LinkedList logGroupList = logGroupProcessor.load("src/test/resources/lookup"); + + // CFE-12 lookups have no way to determine all groups for host, iterating all + String index = null; + for (LogGroup group : logGroupList) { + final String tmp = group.getIndex("example-host-one", + "example-tag-one" + ); + + if (tmp != null) { + index = tmp; + } + } + Assertions.assertEquals("example-tag-one", index); + } +} diff --git a/src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupSearchTest.java b/src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupSearchTest.java new file mode 100644 index 0000000..c3f4924 --- /dev/null +++ b/src/test/java/com/teragrep/pth_05/tests/authz/loggroup/LogGroupSearchTest.java @@ -0,0 +1,75 @@ +/* + * S3 Authorization enabled object gateway service pth_05 + * Copyright (C) 2021 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_05.tests.authz.loggroup; + +import com.teragrep.pth_05.authz.loggroup.LogGroupSearch; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; + +public class LogGroupSearchTest { + + @Test + public void loadTest() throws IOException { + + try (DirectoryStream stream = Files.newDirectoryStream(Paths.get("src/test/resources/lookup"))) { + LogGroupSearch groupSearch = new LogGroupSearch(stream); + + final Set expectedSet = new HashSet<>(); + expectedSet.add("test-data"); + expectedSet.add("example"); + Assertions.assertEquals(expectedSet, groupSearch.getGroupList()); + } + } +} diff --git a/src/test/resources/authorize.json b/src/test/resources/authorize.json new file mode 100644 index 0000000..9b32cf2 --- /dev/null +++ b/src/test/resources/authorize.json @@ -0,0 +1,26 @@ +[ + { + "group": "all-access", + "allowedIndexes": [ + "*" + ] + }, + { + "group": "root", + "allowedIndexes": [ + "testtag" + ] + }, + { + "group": "no-access", + "allowedIndexes": [ + "" + ] + }, + { + "group": "", + "allowedIndexes": [ + "no-group" + ] + } +] diff --git a/src/test/resources/blobstore/hundred-year/2021/07-26/testhost/testtag/testtag.log-2021072608.log.gz b/src/test/resources/blobstore/hundred-year/2021/07-26/testhost/testtag/testtag.log-2021072608.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..5e03ca712af20ae3eba9b2a243f96b45afd6b1b8 GIT binary patch literal 37 ocmb2|=3oE=W^Ij=9p(n18lJj(fgY#Ncs^xfkbc{qB?y!S0GdY$OaK4? literal 0 HcmV?d00001 diff --git a/src/test/resources/credentials.json b/src/test/resources/credentials.json new file mode 100644 index 0000000..e39e090 --- /dev/null +++ b/src/test/resources/credentials.json @@ -0,0 +1 @@ +[{"identity": "trusted-5", "credential": "nPBSgvBZVGRQ2Zu8TKItH9bElr0eaosYC0n6BTPqlok"}, {"identity": "trusted-4", "credential": "UhR6yL4MhtH__Vi-PCC3QRzoHWa4D2WcMvAnziDd_-4"}, {"identity": "trusted-3", "credential": "Uv7fhHVwnmb_5kY6FvClBblhWNcD6cAOQLfrdFNs8ic"}, {"identity": "trusted-2", "credential": "3H9ipjEPe-qjrmlctPMEdrfEOeMYFOzMpBZqvdFfHzA"}, {"identity": "trusted-1", "credential": "LaqEGMxzE4_fh6KvMinAJY7wqzb4jw9SwvpeGjWuAlA"}, {"identity": "trusted-10", "credential": "xFM5aNRR4Xo4xAKvsXdqkNxD3YqglwXv6rL3vgG258s"}, {"identity": "trusted-9", "credential": "QIC-4m2E_km1S4EQTa_hBT1Qhx6PXnbxYTL19rWW10Y"}, {"identity": "trusted-8", "credential": "jY4exWgX6AqVGVmR7jv7Slw7u-k78ZtXH8QlpRfq1OE"}, {"identity": "trusted-7", "credential": "ZBLc0y3qH4Bj3F2J2OdGjlYcNuelxe1p2zi7xQ-EZ1g"}, {"identity": "trusted-6", "credential": "SsvvpF3gYyZ7UDOWyUPm_N7pG7SaMibdzc1NRuHB1GY"}, {"identity": "trusted-15", "credential": "zjT8fqrW_Gg1rcOm2IJtt0MfHrzGHC-X1zuL_khJosM"}, {"identity": "trusted-14", "credential": "VQRCqeCKao6XEtDLutqCECMknAHztnyXKnlXdMMt6nY"}, {"identity": "trusted-13", "credential": "-K4NcyPnHBBw_TyGu6JXBy7GkQ9udsuc_IMXVxRc4EU"}, {"identity": "trusted-12", "credential": "XOsAqIhmKUTwWMjWwDaYmVgR8sl_l70H1oDPBw9z2yY"}, {"identity": "trusted-11", "credential": "AesuBKXPIbRsO3__1RujtGxyht-HCdG6EhkJr_iFkLs"}, {"identity": "trusted-20", "credential": "VMIvvyL7DQKzlhdyE9sYqhv7kQrAtlZt8SGHO_u9YOo"}, {"identity": "trusted-19", "credential": "ZisCM-YuYjPqJKQUfbjykDyDmJkSdOx-uZ5clwjFwR4"}, {"identity": "trusted-18", "credential": "WsfvrJgztuwgyQWciJCpiVn--fe-Ye_8-tzOGQQdETE"}, {"identity": "trusted-17", "credential": "m1WBsyOKejtKYj93C9SOFk8QamznH3xrTWmZyc3mkLY"}, {"identity": "trusted-16", "credential": "L29E_41s2V_lSo2bxgUdnMt3Ghobd5TuwphGRqdlFXg"}, {"identity": "root", "credential": "aP051Xd3f1n3d@account"}] diff --git a/src/test/resources/lookup/example_hosts.json b/src/test/resources/lookup/example_hosts.json new file mode 100644 index 0000000..72440e0 --- /dev/null +++ b/src/test/resources/lookup/example_hosts.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "nomatch": "unknown", + "type": "string", + "table": [ + { + "index": "example-host-one", + "value": true + }, + { + "index": "example-host-two", + "value": true + } + ] +} diff --git a/src/test/resources/lookup/example_indexes.json b/src/test/resources/lookup/example_indexes.json new file mode 100644 index 0000000..466a268 --- /dev/null +++ b/src/test/resources/lookup/example_indexes.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "nomatch": "unknown", + "type": "string", + "table": [ + { + "index": "example-tag-one", + "value": "example-tag-one" + }, + { + "index": "example-tag-two", + "value": "example-tag-two" + } + ] +} diff --git a/src/test/resources/lookup/test-data_hosts.json b/src/test/resources/lookup/test-data_hosts.json new file mode 100644 index 0000000..37a6f07 --- /dev/null +++ b/src/test/resources/lookup/test-data_hosts.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "nomatch": "unknown", + "type": "string", + "table": [ + { + "index": "testhost", + "value": true + }, + { + "index": "dummyhost", + "value": true + } + ] +} diff --git a/src/test/resources/lookup/test-data_indexes.json b/src/test/resources/lookup/test-data_indexes.json new file mode 100644 index 0000000..90b80bc --- /dev/null +++ b/src/test/resources/lookup/test-data_indexes.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "nomatch": "unknown", + "type": "string", + "table": [ + { + "index": "testtag", + "value": "testtag" + }, + { + "index": "dummytag", + "value": "dummytag" + } + ] +} diff --git a/src/test/resources/s3cfg.valid.conf b/src/test/resources/s3cfg.valid.conf new file mode 100644 index 0000000..d80900b --- /dev/null +++ b/src/test/resources/s3cfg.valid.conf @@ -0,0 +1,83 @@ +[default] +access_key = root +access_token = +add_encoding_exts = +add_headers = +bucket_location = US +ca_certs_file = +cache_file = +check_ssl_certificate = True +check_ssl_hostname = True +cloudfront_host = cloudfront.amazonaws.com +connection_pooling = True +content_disposition = +content_type = +default_mime_type = binary/octet-stream +delay_updates = False +delete_after = False +delete_after_fetch = False +delete_removed = False +dry_run = False +enable_multipart = True +encoding = UTF-8 +encrypt = False +expiry_date = +expiry_days = +expiry_prefix = +follow_symlinks = False +force = False +get_continue = False +gpg_command = /usr/bin/gpg +gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_passphrase = +guess_mime_type = True +host_base = localhost:8080 +host_bucket = localhost +human_readable_sizes = False +invalidate_default_index_on_cf = False +invalidate_default_index_root_on_cf = True +invalidate_on_cf = False +kms_key = +limit = -1 +limitrate = 0 +list_md5 = False +log_target_prefix = +long_listing = False +max_delete = -1 +mime_type = +multipart_chunk_size_mb = 15 +multipart_max_chunks = 10000 +preserve_attrs = True +progress_meter = True +proxy_host = +proxy_port = 0 +public_url_use_https = False +put_continue = False +recursive = False +recv_chunk = 65536 +reduced_redundancy = False +requester_pays = False +restore_days = 1 +restore_priority = Standard +secret_key = aP051Xd3f1n3d@account +send_chunk = 65536 +server_side_encryption = False +signature_v2 = False +signurl_use_https = False +simpledb_host = sdb.amazonaws.com +skip_existing = False +socket_timeout = 300 +stats = False +stop_on_error = False +storage_class = +throttle_max = 100 +upload_id = +urlencoding_mode = normal +use_http_expect = False +use_https = False +use_mime_magic = True +verbosity = WARNING +website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/ +website_error = +website_index = index.html diff --git a/src/test/resources/test.properties b/src/test/resources/test.properties new file mode 100644 index 0000000..98e8e3b --- /dev/null +++ b/src/test/resources/test.properties @@ -0,0 +1,6 @@ +pth_05.endpoint=http://127.0.0.1:8080 +pth_05.credentials.file=src/test/resources/credentials.json +pth_05.authorize.file=src/test/resources/authorize.json +pth_05.lookup.path=src/test/resources/lookup +jclouds.filesystem.basedir=src/test/resources/blobstore +jclouds.provider=filesystem