Skip to content

Commit

Permalink
Merge pull request #43543 from LakshanWeerasinghe/load-libs-with-graphql
Browse files Browse the repository at this point in the history 
Load ballerina, ballerinax package meta using Central GraphQL API
  • Loading branch information
@LakshanWeerasinghe
LakshanWeerasinghe authored Nov 20, 2024
2 parents fd736a7 + deeb60f commit c4c87cb
Show file tree
Hide file tree
Showing 17 changed files with 513 additions and 647 deletions.
44 changes: 20 additions & 24 deletions cli/central-client/src/main/java/org/ballerinalang/central/client/CentralAPIClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import java.io.IOException;
import java.io.PrintStream;
import java.net.Proxy;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -140,6 +141,8 @@ public class CentralAPIClient {
private static final String ERR_PACKAGE_UN_DEPRECATE = "error: failed to undo deprecation of the package: ";
private static final String ERR_PACKAGE_RESOLUTION = "error: while connecting to central: ";
private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8");
private static final MediaType JSON_CONTENT_TYPE = MediaType.parse("application/json");

// System property name for enabling central verbose
public static final String SYS_PROP_CENTRAL_VERBOSE_ENABLED = "CENTRAL_VERBOSE_ENABLED";
private static final int DEFAULT_CONNECT_TIMEOUT = 60;
Expand Down Expand Up @@ -1346,52 +1349,45 @@ public void deprecatePackage(String packageInfo, String deprecationMsg, String s
}

/**
* Get packages from central.
* Get packages information using graphql API.
*
* @param params Search query param map.
* @param supportedPlatform The supported platform.
* @param ballerinaVersion The ballerina version.
* @return Package list
* @throws CentralClientException Central client exception.
* @param query payload query
* @param supportedPlatform supported platform
* @param ballerinaVersion ballerina version
* @return {@link JsonElement} Json Response
*/
public JsonElement getPackages(Map<String, String> params, String supportedPlatform, String ballerinaVersion)
public JsonElement getCentralPackagesUsingGraphQL(String query, String supportedPlatform, String ballerinaVersion)
throws CentralClientException {
Optional<ResponseBody> body = Optional.empty();
OkHttpClient client = new OkHttpClient.Builder()
.followRedirects(false)
.connectTimeout(connectTimeout, TimeUnit.SECONDS)
.connectTimeout(callTimeout, TimeUnit.SECONDS)
.readTimeout(readTimeout, TimeUnit.SECONDS)
.writeTimeout(writeTimeout, TimeUnit.SECONDS)
.callTimeout(callTimeout, TimeUnit.SECONDS)
.proxy(this.proxy)
.build();

try {
HttpUrl.Builder httpBuilder = Objects.requireNonNull(HttpUrl.parse(this.baseUrl))
.newBuilder().addPathSegment(PACKAGES);
for (Map.Entry<String, String> param : params.entrySet()) {
httpBuilder.addQueryParameter(param.getKey(), param.getValue());
}

Request searchReq = getNewRequest(supportedPlatform, ballerinaVersion)
.get()
.url(httpBuilder.build())
HttpUrl.Builder httpUrl = Objects.requireNonNull(HttpUrl.parse(this.baseUrl)).newBuilder();
Request request = getNewRequest(supportedPlatform, ballerinaVersion)
.url(httpUrl.build())
.post(RequestBody.create(JSON_CONTENT_TYPE, query.getBytes(StandardCharsets.UTF_8)))
.build();

Call httpRequestCall = client.newCall(searchReq);
Response searchResponse = httpRequestCall.execute();

ResponseBody responseBody = searchResponse.body();
Call httpRequest = client.newCall(request);
Response response = httpRequest.execute();
ResponseBody responseBody = response.body();
body = responseBody != null ? Optional.of(responseBody) : Optional.empty();
if (body.isPresent()) {
MediaType contentType = body.get().contentType();
if (contentType != null && isApplicationJsonContentType(contentType.toString()) &&
searchResponse.code() == HttpsURLConnection.HTTP_OK) {
response.code() == HttpsURLConnection.HTTP_OK) {
return new Gson().toJsonTree(body.get().string());
}
}
handleResponseErrors(searchResponse, ERR_CANNOT_SEARCH);
return new JsonArray();
handleResponseErrors(response, ERR_CANNOT_SEARCH);
return new JsonObject();
} catch (IOException e) {
throw new CentralClientException(ERR_CANNOT_SEARCH + "'. Reason: " + e.getMessage());
} finally {
Expand Down
17 changes: 17 additions & 0 deletions compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/util/RepoUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ public final class RepoUtils {
private static final String PRODUCTION_URL = "https://api.central.ballerina.io/2.0/registry";
private static final String STAGING_URL = "https://api.staging-central.ballerina.io/2.0/registry";
private static final String DEV_URL = "https://api.dev-central.ballerina.io/2.0/registry";
private static final String PRODUCTION_GRAPHQL_URL = "https://api.central.ballerina.io/2.0/graphql";
private static final String STAGING_GRAPHQL_URL = "https://api.staging-central.ballerina.io/2.0/graphql";
private static final String DEV_GRAPHQL_URL = "https://api.dev-central.ballerina.io/2.0/graphql";

private static final String BALLERINA_ORG = "ballerina";
private static final String BALLERINAX_ORG = "ballerinax";
Expand Down Expand Up @@ -158,6 +161,20 @@ public static String getRemoteRepoURL() {
return PRODUCTION_URL;
}

/**
* Get the graphQL remote repo URL.
*
* @return URL of the remote repository
*/
public static String getRemoteRepoGraphQLURL() {
if (SET_BALLERINA_STAGE_CENTRAL) {
return STAGING_GRAPHQL_URL;
} else if (SET_BALLERINA_DEV_CENTRAL) {
return DEV_GRAPHQL_URL;
}
return PRODUCTION_GRAPHQL_URL;
}

/**
* Get the staging URL.
*
Expand Down
109 changes: 20 additions & 89 deletions ...erver-core/src/main/java/org/ballerinalang/langserver/CentralPackageDescriptorLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,15 @@

import com.google.gson.Gson;
import com.google.gson.JsonElement;
import io.ballerina.projects.PackageName;
import io.ballerina.projects.PackageOrg;
import io.ballerina.projects.PackageVersion;
import io.ballerina.projects.Settings;
import io.ballerina.projects.util.ProjectUtils;
import org.ballerinalang.central.client.CentralAPIClient;
import org.ballerinalang.central.client.model.Package;
import org.ballerinalang.langserver.commons.LanguageServerContext;
import org.ballerinalang.langserver.extensions.ballerina.connector.CentralPackageListResult;
import org.wso2.ballerinalang.util.RepoUtils;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;

/**
Expand All @@ -43,6 +37,8 @@ public class CentralPackageDescriptorLoader {
public static final LanguageServerContext.Key<CentralPackageDescriptorLoader> CENTRAL_PACKAGE_HOLDER_KEY =
new LanguageServerContext.Key<>();
private final List<LSPackageLoader.ModuleInfo> centralPackages = new ArrayList<>();
private static final String GET_PACKAGES_QUERY =
"{\"query\": \"{packages(orgName:\\\"%s\\\" limit: %s) {packages {name version organization}}}\"}";
private boolean isLoaded = false;

private final LSClientLogger clientLogger;
Expand All @@ -65,103 +61,38 @@ public CompletableFuture<List<LSPackageLoader.ModuleInfo>> getCentralPackages()
if (!isLoaded) {
//Load packages from central
clientLogger.logTrace("Loading packages from Ballerina Central");
this.getPackagesFromCentral().forEach(packageInfo -> {
PackageOrg packageOrg = PackageOrg.from(packageInfo.getOrganization());
PackageName packageName = PackageName.from(packageInfo.getName());
PackageVersion packageVersion = PackageVersion.from(packageInfo.getVersion());
centralPackages.add(new LSPackageLoader.ModuleInfo(packageOrg, packageName, packageVersion, null));
});
centralPackages.addAll(this.getCentralGraphQLPackages());
clientLogger.logTrace("Successfully loaded packages from Ballerina Central");
}
isLoaded = true;
return this.centralPackages;
});
}

private List<Package> getPackagesFromCentral() {
List<Package> packageList = new ArrayList<>();
private List<LSPackageLoader.ModuleInfo> getCentralGraphQLPackages() {
try {
for (int page = 0;; page++) {
Settings settings = RepoUtils.readSettings();

CentralAPIClient centralAPIClient = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
ProjectUtils.initializeProxy(settings.getProxy()), ProjectUtils.getAccessTokenOfCLI(settings));
CentralPackageDescriptor descriptor = new CentralPackageDescriptor("ballerinax", 10, page * 10);

JsonElement newClientConnectors = centralAPIClient.getPackages(descriptor.getQueryMap(),
"any", RepoUtils.getBallerinaVersion());

CentralPackageListResult packageListResult = new Gson().fromJson(newClientConnectors.getAsString(),
CentralPackageListResult.class);
packageList.addAll(packageListResult.getPackages());
int listResultCount = packageListResult.getCount();

if (packageList.size() == listResultCount || descriptor.getOffset() >= listResultCount) {
break;
}
}
Settings settings = RepoUtils.readSettings();
CentralAPIClient centralAPIClient = new CentralAPIClient(RepoUtils.getRemoteRepoGraphQLURL(),
ProjectUtils.initializeProxy(settings.getProxy()), ProjectUtils.getAccessTokenOfCLI(settings));
String query = String.format(GET_PACKAGES_QUERY, "ballerinax", 800);
JsonElement allPackagesResponse = centralAPIClient.getCentralPackagesUsingGraphQL(query, "any",
RepoUtils.getBallerinaVersion());
CentralPackageGraphQLResponse response = new Gson().fromJson(allPackagesResponse.getAsString(),
CentralPackageGraphQLResponse.class);
return response.data.packages.packages;

} catch (Exception e) {
// ignore
}
return packageList;
return Collections.emptyList();
}

/**
* Central package descriptor.
*/
public static class CentralPackageDescriptor {
private String organization;
private int limit;
private int offset;

public CentralPackageDescriptor(String organization, int limit, int offset) {
this.organization = organization;
this.limit = limit;
this.offset = offset;
}

public String getOrganization() {
return organization;
}

public void setOrganization(String organization) {
this.organization = organization;
}

public int getLimit() {
return limit;
}

public void setLimit(int limit) {
this.limit = limit;
}

public int getOffset() {
return offset;
}

public void setOffset(int offset) {
this.offset = offset;
}

public Map<String, String> getQueryMap() {
Map<String, String> params = new HashMap<>();
params.put("readme", "false");

if (getOrganization() != null) {
params.put("org", getOrganization());
}

if (getLimit() != 0) {
params.put("limit", Integer.toString(getLimit()));
}
public record CentralPackageGraphQLResponse(Packages data) {
}

if (getOffset() != 0) {
params.put("offset", Integer.toString(getOffset()));
}
public record Packages(PackageList packages) {
}

return params;
}
public record PackageList(List<LSPackageLoader.ModuleInfo> packages) {
}
}
58 changes: 42 additions & 16 deletions ...r/modules/langserver-core/src/main/java/org/ballerinalang/langserver/LSPackageLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
*/
package org.ballerinalang.langserver;

import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
import io.ballerina.compiler.api.ModuleID;
import io.ballerina.compiler.api.SemanticModel;
import io.ballerina.projects.Module;
Expand Down Expand Up @@ -263,16 +265,16 @@ public List<ModuleInfo> getAllVisiblePackages(DocumentServiceContext ctx) {
packageInstance.packageVersion(), packageInstance.project().sourceRoot());
moduleInfo.setModuleFromCurrentPackage(true);
Optional<Module> currentModule = ctx.currentModule();
String packageName = moduleInfo.packageName().value();
String packageName = moduleInfo.packageName();
String moduleName = module.descriptor().name().moduleNamePart();
String qualifiedModName = packageName + Names.DOT + moduleName;
if (currentModule.isEmpty() || module.isDefaultModule() || module.equals(currentModule.get()) ||
ModuleUtil.matchingImportedModule(ctx, "", qualifiedModName).isPresent()) {
return;
} else {
moduleInfo.packageName = PackageName.from(packageName + "." + moduleName);
moduleInfo.packageName = packageName + "." + moduleName;
}
packagesList.put(moduleInfo.packageName.value(), moduleInfo);
packagesList.put(moduleInfo.packageName, moduleInfo);
});
return new ArrayList<>(packagesList.values());
}
Expand Down Expand Up @@ -350,35 +352,59 @@ public List<ModuleInfo> updatePackageMap(DocumentServiceContext context) {
return moduleInfos;
}

/**
* A light-weight package information holder.
*/
/**
* A light-weight package information holder.
*/
public static class ModuleInfo {

private final PackageOrg packageOrg;
private PackageName packageName;
private final PackageVersion packageVersion;
private static final String JSON_PROPERTY_ORGANIZATION = "organization";
@SerializedName(JSON_PROPERTY_ORGANIZATION)
private final String packageOrg;

private static final String JSON_PROPERTY_NAME = "name";
@SerializedName(JSON_PROPERTY_NAME)
private String packageName;

private static final String JSON_PROPERTY_VERSION = "version";
@SerializedName(JSON_PROPERTY_VERSION)
private final String packageVersion;

@Expose(deserialize = false)
private final Path sourceRoot;

@Expose(deserialize = false)
private final String moduleIdentifier;

@Expose(deserialize = false)
private boolean isModuleFromCurrentPackage = false;

@Expose(deserialize = false)
private final List<ServiceTemplateGenerator.ListenerMetaData> listenerMetaData = new ArrayList<>();

public ModuleInfo(PackageOrg packageOrg, PackageName packageName, PackageVersion version, Path path) {
public ModuleInfo(String packageOrg, String packageName, String packageVersion) {
this.packageOrg = packageOrg;
this.packageName = packageName;
this.packageVersion = version;
this.packageVersion = packageVersion;
this.sourceRoot = null;
this.moduleIdentifier = packageOrg + "/" + packageName;
}

public ModuleInfo(PackageOrg packageOrg, PackageName packageName, PackageVersion version, Path path) {
this.packageOrg = packageOrg.value();
this.packageName = packageName.value();
this.packageVersion = version.value().toString();
this.sourceRoot = path;
this.moduleIdentifier = packageOrg.toString().isEmpty() ? packageName.toString() :
packageOrg + "/" + packageName.toString();
packageOrg + "/" + packageName;
}

public ModuleInfo(Package pkg) {
this.packageOrg = pkg.packageOrg();
this.packageName = pkg.packageName();
this.packageVersion = pkg.packageVersion();
this.packageOrg = pkg.packageOrg().value();
this.packageName = pkg.packageName().value();
this.packageVersion = pkg.packageVersion().value().toString();
this.sourceRoot = pkg.project().sourceRoot();
this.moduleIdentifier = packageOrg.toString() + "/" + packageName.toString();
addServiceTemplateMetaData();
Expand All @@ -400,15 +426,15 @@ public void setModuleFromCurrentPackage(boolean moduleFromCurrentPackage) {
isModuleFromCurrentPackage = moduleFromCurrentPackage;
}

public PackageName packageName() {
public String packageName() {
return packageName;
}

public PackageOrg packageOrg() {
public String packageOrg() {
return packageOrg;
}

public PackageVersion packageVersion() {
public String packageVersion() {
return packageVersion;
}

Expand All @@ -421,7 +447,7 @@ public String packageIdentifier() {
}

private void addServiceTemplateMetaData() {
String orgName = ModuleUtil.escapeModuleName(this.packageOrg().value());
String orgName = ModuleUtil.escapeModuleName(this.packageOrg());
Project project = ProjectLoader.loadProject(this.sourceRoot());
//May take some time as we are compiling projects.
PackageCompilation packageCompilation = project.currentPackage().getCompilation();
Expand Down
Loading

0 comments on commit c4c87cb

Please sign in to comment.