Skip to content

Commit

Permalink
Merge pull request #433 from akto-api-security/develop
Browse files Browse the repository at this point in the history 
fixed execute section
  • Loading branch information
@avneesh-akto
avneesh-akto authored Aug 16, 2023
2 parents 75d285f + 522e4cd commit 1873215
Show file tree
Hide file tree
Showing 248 changed files with 12,356 additions and 3,313 deletions.
42 changes: 34 additions & 8 deletions .github/workflows/csvUpdate.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
REPOSITORY_OWNER = 'akto-api-security'
REPOSITORY_NAME = 'akto'
BRANCH_NAME = 'master'
BRANCH_NAME = '${{ github.ref_name }}'
FOLDER_PATH = 'apps/dashboard/src/main/resources/inbuilt_test_yaml_files'
FILE_EXTENSION = '.yaml'
Expand All @@ -59,9 +59,12 @@ jobs:
repository = g.get_repo(f"{REPOSITORY_OWNER}/{REPOSITORY_NAME}")
try:
content_of_file = repository.get_contents(file_path)
data = content_of_file.decoded_content.decode("utf-8")
print("trying to download: ", yaml_url)
rr = requests.get(yaml_url, allow_redirects=True)
data = rr.content
print("received: ", data)
yaml_data = yaml.safe_load(data)
print(f"Successfully fetched the template: {repository_parts[-1]}")
return yaml_data, data
except Exception as e:
Expand All @@ -79,26 +82,49 @@ jobs:
column.append(yaml_data['info']['impact'])
column.append(yaml_data['info']['category']['displayName'])
column.append(yaml_data['info']['severity'])
print("column: ", column)
references = []
if 'references' in yaml_data['info']:
column.append(yaml_data['info']['references'])
references = yaml_data['info']['references']
if references is None:
references = []

else:
column.append("")

print("references: ", references)
column.append(yaml_content)
url_path = ''
column.append(url_path)
column.append(yaml_data['id'])
rlink1 = "" if len(references) <= 0 else references[0]
rlink2 = "" if len(references) <= 1 else references[1]
rlink3 = "" if len(references) <= 2 else references[2]
rlink1 = ""
if len(references) >= 1:
rlink1 = references[0]
print("rlink1", rlink1)


rlink2 = ""

if len(references) >= 2:
rlink2 = references[1]

print("rlink2", rlink2)


rlink3 = ""

if len(references) >= 3:
rlink3 = references[2]

print("rlink3", rlink3)


column.append(rlink1)
column.append(rlink2)
column.append(rlink3)
Expand Down Expand Up @@ -128,7 +154,7 @@ jobs:
writer.writerow(['Name', 'Slug', 'Description', 'Details', 'Impact', 'OWASP Category', 'Severity', 'References', 'Content', 'URL Path', 'Test_ID', 'RLink1', 'RLink2', 'RLink3'])

for yaml_file in yaml_files:
yaml_url = yaml_file['html_url']
yaml_url = yaml_file['download_url']
yaml_data, yaml_content = fetch_yaml_data(access_token, yaml_url)
if yaml_data:
write_to_csv(writer, yaml_data, yaml_content)
Expand Down
11 changes: 4 additions & 7 deletions .github/workflows/testsigma.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,8 @@ name: Testsigma
on:
workflow_dispatch:
inputs:
executionId:
default: "208"
environmentId:
default: "8"
testPlanId:
default: "361"

jobs:
build:
Expand All @@ -15,6 +13,5 @@ jobs:
steps:
- name: Trigger Testsigma Test
run: |
curl -X POST -H "Content-type: application/json" -H "Accept:application/json" -H "Authorization: Bearer ${{ secrets.TESTSIGMA_API_KEY }}" \
https://app.testsigma.com/api/v1/execution_results -d "{\"executionId\": \"${{ github.event.inputs.executionId }}\", \
\"environmentId\": \"${{ github.event.inputs.environmentId }}\"}"
curl --insecure -X POST -H "Content-type: application/json" -H "Accept:application/json" -H "Authorization: Bearer ${{ vars.TESTSIGMA_API_KEY }}" \
${{vars.TS_DASHBOARD_URL}}/api/v1/test_plan_results -d "{\"testPlanId\": \"${{ github.event.inputs.testPlanId }}\"}"
4 changes: 2 additions & 2 deletions apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,9 @@ public APICatalogSync(String userIdentifier,int thresh) {
mergeAsyncOutside = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()).getMergeAsyncOutside();
}
} catch (Exception e) {

}

}

public static final int STRING_MERGING_THRESHOLD = 10;
Expand Down
52 changes: 32 additions & 20 deletions apps/api-runtime/src/main/java/com/akto/runtime/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,14 @@
import com.akto.dto.test_editor.TestConfig;
import com.akto.dto.test_editor.YamlTemplate;
import com.akto.runtime.policies.AktoPolicies;
import com.akto.util.AccountTask;
import com.google.gson.Gson;
import com.mongodb.ConnectionString;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.UpdateOptions;
import com.mongodb.client.model.Updates;

import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.errors.WakeupException;
import org.apache.kafka.common.serialization.StringDeserializer;
Expand All @@ -44,6 +46,7 @@ public class Main {
public static final String GROUP_NAME = "group_name";
public static final String VXLAN_ID = "vxlanId";
public static final String VPC_CIDR = "vpc_cidr";
public static final String ACCOUNT_ID = "account_id";
private static final Logger logger = LoggerFactory.getLogger(Main.class);
private static final LoggerMaker loggerMaker = new LoggerMaker(Main.class);

Expand All @@ -66,8 +69,8 @@ public static boolean tryForCollectionName(String message) {
Map<String, Object> json = gson.fromJson(message, Map.class);

// logger.info("Json size: " + json.size());
boolean withoutCidrCond = json.size() == 2 && json.containsKey(GROUP_NAME) && json.containsKey(VXLAN_ID);
boolean withCidrCond = json.size() == 3 && json.containsKey(GROUP_NAME) && json.containsKey(VXLAN_ID) && json.containsKey(VPC_CIDR);
boolean withoutCidrCond = json.containsKey(GROUP_NAME) && json.containsKey(VXLAN_ID);
boolean withCidrCond = json.containsKey(GROUP_NAME) && json.containsKey(VXLAN_ID) && json.containsKey(VPC_CIDR);
if (withCidrCond || withoutCidrCond) {
ret = true;
String groupName = (String) (json.get(GROUP_NAME));
Expand All @@ -78,9 +81,13 @@ public static boolean tryForCollectionName(String message) {
Updates.set(ApiCollection.NAME, groupName)
);

if (json.size() == 3) {
if (json.containsKey(VPC_CIDR)) {
List<String> cidrList = (List<String>) json.get(VPC_CIDR);
logger.info("cidrList: " + cidrList);
// For old deployments, we won't receive ACCOUNT_ID. If absent, we assume 1_000_000.
String accountIdStr = (String) (json.get(ACCOUNT_ID));
int accountId = StringUtils.isNumeric(accountIdStr) ? Integer.parseInt(accountIdStr) : 1_000_000;
Context.accountId.set(accountId);
AccountSettingsDao.instance.getMCollection().updateOne(
AccountSettingsDao.generateFilter(), Updates.addEachToSet("privateCidrList", cidrList), new UpdateOptions().upsert(true)
);
Expand All @@ -105,18 +112,21 @@ public static void insertRuntimeFilters() {
}

public static Kafka kafkaProducer = null;
private static void buildKafka(int accountId) {
Context.accountId.set(accountId);
AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter(accountId));
if (accountSettings != null && accountSettings.getCentralKafkaIp()!= null) {
String centralKafkaBrokerUrl = accountSettings.getCentralKafkaIp();
int centralKafkaBatchSize = AccountSettings.DEFAULT_CENTRAL_KAFKA_BATCH_SIZE;
int centralKafkaLingerMS = AccountSettings.DEFAULT_CENTRAL_KAFKA_LINGER_MS;
if (centralKafkaBrokerUrl != null) {
kafkaProducer = new Kafka(centralKafkaBrokerUrl, centralKafkaLingerMS, centralKafkaBatchSize);
logger.info("Connected to central kafka @ " + Context.now());
private static void buildKafka() {
logger.info("Building kafka...................");
AccountTask.instance.executeTask(t -> {
int accountId = Context.accountId.get();
AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter(accountId));
if (accountSettings != null && accountSettings.getCentralKafkaIp()!= null) {
String centralKafkaBrokerUrl = accountSettings.getCentralKafkaIp();
int centralKafkaBatchSize = AccountSettings.DEFAULT_CENTRAL_KAFKA_BATCH_SIZE;
int centralKafkaLingerMS = AccountSettings.DEFAULT_CENTRAL_KAFKA_LINGER_MS;
if (centralKafkaBrokerUrl != null) {
kafkaProducer = new Kafka(centralKafkaBrokerUrl, centralKafkaLingerMS, centralKafkaBatchSize);
logger.info("Connected to central kafka @ " + Context.now());
}
}
}
}, "build-kafka-task");
}

public static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2);
Expand Down Expand Up @@ -157,19 +167,16 @@ public static void main(String[] args) {

if (topicName == null) topicName = "akto.api.logs";

// mongoURI = "mongodb://write_ops:write_ops@cluster0-shard-00-00.yg43a.mongodb.net:27017,cluster0-shard-00-01.yg43a.mongodb.net:27017,cluster0-shard-00-02.yg43a.mongodb.net:27017/myFirstDatabase?ssl=true&replicaSet=atlas-qd3mle-shard-0&authSource=admin&retryWrites=true&w=majority";
DaoInit.init(new ConnectionString(mongoURI));
Context.accountId.set(1_000_000);
initializeRuntime();

String centralKafkaTopicName = AccountSettings.DEFAULT_CENTRAL_KAFKA_TOPIC_NAME;

int accountIdHardcoded = Context.accountId.get();
buildKafka(accountIdHardcoded);
buildKafka();
scheduler.scheduleAtFixedRate(new Runnable() {
public void run() {
if (kafkaProducer == null || !kafkaProducer.producerReady) {
buildKafka(accountIdHardcoded);
buildKafka();
}
}
}, 5, 5, TimeUnit.MINUTES);
Expand Down Expand Up @@ -335,9 +342,14 @@ public void run() {
}

public static void initializeRuntime(){
AccountTask.instance.executeTask(t -> {
initializeRuntimeHelper();
}, "initialize-runtime-task");
}

public static void initializeRuntimeHelper() {
SingleTypeInfoDao.instance.getMCollection().updateMany(Filters.exists("apiCollectionId", false), Updates.set("apiCollectionId", 0));
SingleTypeInfo.init();

createIndices();
insertRuntimeFilters();
try {
Expand Down
2 changes: 1 addition & 1 deletion apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public void main(List<HttpResponseParams> httpResponseParamsList, boolean syncNo
}

public void process(HttpResponseParams httpResponseParams) throws Exception {
List<CustomAuthType> customAuthTypes = SingleTypeInfo.activeCustomAuthTypes;
List<CustomAuthType> customAuthTypes = SingleTypeInfo.getCustomAuthType(Integer.parseInt(httpResponseParams.getAccountId()));
ApiInfo.ApiInfoKey apiInfoKey = ApiInfo.ApiInfoKey.generateFromHttpResponseParams(httpResponseParams);
PolicyCatalog policyCatalog = getApiInfoFromMap(apiInfoKey);
ApiInfo apiInfo = policyCatalog.getApiInfo();
Expand Down
24 changes: 15 additions & 9 deletions apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.akto.dto.User;
import com.akto.dto.runtime_filters.RuntimeFilter;
import com.akto.dto.traffic.SampleData;
import com.akto.dto.type.AccountDataTypesInfo;
import com.akto.dto.type.RequestTemplate;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.dto.type.URLTemplate;
Expand Down Expand Up @@ -46,15 +47,20 @@ public void changeAccountId() {
}

public void testInitializer(){
SingleTypeInfo.aktoDataTypeMap = new HashMap<>();
SingleTypeInfo.aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
Map<String, AktoDataType> aktoDataTypeMap = new HashMap<>();
aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID);
if (info == null) {
info = new AccountDataTypesInfo();
}
info.setAktoDataTypeMap(aktoDataTypeMap);
SingleTypeInfo.getAccountToDataTypesInfo().put(ACCOUNT_ID, info);
}

@Test
Expand Down
26 changes: 17 additions & 9 deletions apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.Map;
import java.util.Set;

import com.akto.dao.context.Context;
import com.akto.dto.type.*;
import com.akto.dto.type.SingleTypeInfo.SubType;
import com.akto.dto.type.SingleTypeInfo.SuperType;
Expand All @@ -27,17 +28,24 @@
import org.junit.Test;

public class TestDump2 {
private final int ACCOUNT_ID = 1_000_000;

public void testInitializer(){
SingleTypeInfo.aktoDataTypeMap = new HashMap<>();
SingleTypeInfo.aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
SingleTypeInfo.aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
Context.accountId.set(ACCOUNT_ID);
Map<String, AktoDataType> aktoDataTypeMap = new HashMap<>();
aktoDataTypeMap.put("JWT", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("PHONE_NUMBER", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("CREDIT_CARD", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("IP_ADDRESS", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("EMAIL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("SSN", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("UUID", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>())));
aktoDataTypeMap.put("URL", new AktoDataType(null, false, null, 0, new IgnoreData(new HashMap<>(), new HashSet<>()))); AccountDataTypesInfo info = SingleTypeInfo.getAccountToDataTypesInfo().get(ACCOUNT_ID);
if (info == null) {
info = new AccountDataTypesInfo();
}
info.setAktoDataTypeMap(aktoDataTypeMap);
SingleTypeInfo.getAccountToDataTypesInfo().put(ACCOUNT_ID, info);
}
public static String createSimpleResponsePayload() {
BasicDBObject ret = new BasicDBObject();
Expand Down
Loading

0 comments on commit 1873215

Please sign in to comment.