Merge pull request #672 from modelix/fix/bulk-sync-pull-caching

MODELIX-740 bulk-model-sync should not cache pull result from model-server
modelix · Apr 12, 2024 · 5de2880 · 5de2880
2 parents 2f7a632 + 6ecbff6
commit 5de2880
Show file tree

Hide file tree

Showing 6 changed files with 157 additions and 76 deletions.
diff --git a/...l-sync-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/ModelSyncGradlePlugin.kt b/...l-sync-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/ModelSyncGradlePlugin.kt
@@ -29,6 +29,7 @@ import org.modelix.model.sync.bulk.gradle.config.ServerSource
 import org.modelix.model.sync.bulk.gradle.config.ServerTarget
 import org.modelix.model.sync.bulk.gradle.config.SyncDirection
 import org.modelix.model.sync.bulk.gradle.tasks.ExportFromModelServer
+import org.modelix.model.sync.bulk.gradle.tasks.GetRevisionInfo
 import org.modelix.model.sync.bulk.gradle.tasks.ImportIntoModelServer
 import org.modelix.model.sync.bulk.gradle.tasks.ValidateSyncSettings
 import java.io.File
@@ -77,7 +78,7 @@ class ModelSyncGradlePlugin : Plugin<Project> {
         val baseDir = project.layout.buildDirectory.dir("model-sync").get().asFile.apply { mkdirs() }
         val jsonDir = baseDir.resolve(syncDirection.name).apply { mkdir() }
         val sourceTask = when (syncDirection.source) {
-            is LocalSource -> registerTasksForLocalSource(syncDirection, project, previousTask, jsonDir)
+            is LocalSource -> registerTasksForLocalSource(syncDirection, previousTask, jsonDir)
             is ServerSource -> registerTasksForServerSource(syncDirection, project, previousTask, jsonDir)
             else -> error("Unknown sync direction source")
         }
@@ -95,15 +96,32 @@ class ModelSyncGradlePlugin : Plugin<Project> {
         jsonDir: File,
     ): TaskProvider<*> {
         val serverSource = syncDirection.source as ServerSource
+        val revisionFile = jsonDir.resolve(".modelix_revision").also { it.createNewFile() }
 
-        val name = "${syncDirection.name}ExportFromModelServer"
-        val exportFromModelServer = project.tasks.register(name, ExportFromModelServer::class.java) {
+        val getRevisionInfo = project.tasks.register(
+            "${syncDirection.name}GetRevisionInfo",
+            GetRevisionInfo::class.java,
+        ) {
             it.dependsOn(previousTask)
-            it.outputDir.set(jsonDir)
-            it.url.set(serverSource.url)
+
+            it.serverUrl.set(serverSource.url)
+            it.revision.set(serverSource.revision)
             it.repositoryId.set(serverSource.repositoryId)
             it.branchName.set(serverSource.branchName)
-            it.revision.set(serverSource.revision)
+            it.revisionFile.set(revisionFile)
+
+            it.outputs.upToDateWhen { serverSource.revision != null }
+        }
+
+        val exportFromModelServer = project.tasks.register(
+            "${syncDirection.name}ExportFromModelServer",
+            ExportFromModelServer::class.java,
+        ) {
+            it.dependsOn(getRevisionInfo)
+            it.url.set(serverSource.url)
+            it.repositoryId.set(serverSource.repositoryId)
+            it.revisionFile.set(revisionFile)
+            it.outputDir.set(jsonDir)
             it.includedModules.set(syncDirection.includedModules)
             it.includedModulePrefixes.set(syncDirection.includedModulePrefixes)
             it.requestTimeoutSeconds.set(serverSource.requestTimeoutSeconds)
@@ -113,7 +131,6 @@ class ModelSyncGradlePlugin : Plugin<Project> {
 
     private fun registerTasksForLocalSource(
         syncDirection: SyncDirection,
-        project: Project,
         previousTask: TaskProvider<*>,
         jsonDir: File,
     ): TaskProvider<*> {
@@ -212,10 +229,6 @@ class ModelSyncGradlePlugin : Plugin<Project> {
         return project.layout.buildDirectory.dir("model-sync").get().asFile
     }
 
-    private fun getDependenciesDir(project: Project): File {
-        return getBaseDir(project).resolve("dependencies")
-    }
-
     private fun readModelixCoreVersion(): String? {
         val resources = javaClass.classLoader.getResources("modelix.core.version.properties") ?: return null
         if (resources.hasMoreElements()) {

diff --git a/...-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ExportFromModelServer.kt b/...-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ExportFromModelServer.kt
@@ -19,66 +19,68 @@ package org.modelix.model.sync.bulk.gradle.tasks
 import kotlinx.coroutines.runBlocking
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.DirectoryProperty
-import org.gradle.api.model.ObjectFactory
+import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.provider.SetProperty
 import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFile
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.OutputDirectory
 import org.gradle.api.tasks.TaskAction
 import org.modelix.model.api.BuiltinLanguages
 import org.modelix.model.api.INode
 import org.modelix.model.api.PBranch
 import org.modelix.model.api.getRootNode
-import org.modelix.model.client2.ModelClientV2
 import org.modelix.model.client2.ModelClientV2PlatformSpecificBuilder
-import org.modelix.model.lazy.BranchReference
 import org.modelix.model.lazy.RepositoryId
 import org.modelix.model.sync.bulk.ModelExporter
 import org.modelix.model.sync.bulk.isModuleIncluded
-import javax.inject.Inject
 import kotlin.time.Duration.Companion.seconds
 
-abstract class ExportFromModelServer @Inject constructor(of: ObjectFactory) : DefaultTask() {
+abstract class ExportFromModelServer : DefaultTask() {
 
-    @Input
-    val url: Property<String> = of.property(String::class.java)
+    @get:Input
+    abstract val url: Property<String>
 
-    @Input
-    @Optional
-    val repositoryId: Property<String> = of.property(String::class.java)
+    @get:Input
+    @get:Optional
+    abstract val repositoryId: Property<String>
 
-    @Input
-    @Optional
-    val branchName: Property<String> = of.property(String::class.java)
+    @get:InputFile
+    abstract val revisionFile: RegularFileProperty
 
-    @Input
-    @Optional
-    val revision: Property<String> = of.property(String::class.java)
+    @get:OutputDirectory
+    abstract val outputDir: DirectoryProperty
 
-    @OutputDirectory
-    val outputDir: DirectoryProperty = of.directoryProperty()
+    @get:Input
+    abstract val includedModules: SetProperty<String>
 
-    @Input
-    val includedModules: SetProperty<String> = of.setProperty(String::class.java)
+    @get:Input
+    abstract val includedModulePrefixes: SetProperty<String>
 
-    @Input
-    val includedModulePrefixes: SetProperty<String> = of.setProperty(String::class.java)
-
-    @Input
-    val requestTimeoutSeconds: Property<Int> = of.property(Int::class.java)
-
-    private fun getBranchReference(): BranchReference = RepositoryId(repositoryId.get()).getBranchReference(branchName.get())
+    @get:Input
+    abstract val requestTimeoutSeconds: Property<Int>
 
     @TaskAction
     fun export() = runBlocking {
         val modelClient = ModelClientV2PlatformSpecificBuilder()
             .url(url.get())
             .requestTimeout(requestTimeoutSeconds.get().seconds)
             .build()
+
+        val revision = revisionFile.get().asFile.readText()
+
         modelClient.use { client ->
             client.init()
-            val branch = loadDataAsBranch(client)
+            val version = if (repositoryId.isPresent) {
+                client.loadVersion(RepositoryId(repositoryId.get()), revision, null)
+            } else {
+                logger.warn("Specifying a repositoryId will be mandatory in the future.")
+                client.loadVersion(revision, null)
+            }
+
+            val branch = PBranch(version.getTree(), client.getIdGenerator())
+
             branch.runRead {
                 val root = branch.getRootNode()
                 logger.info("Got root node: {}", root)
@@ -93,16 +95,6 @@ abstract class ExportFromModelServer @Inject constructor(of: ObjectFactory) : De
         }
     }
 
-    private suspend fun loadDataAsBranch(client: ModelClientV2): PBranch {
-        val version = if (revision.isPresent) {
-            client.loadVersion(revision.get(), null)
-        } else {
-            client.pull(getBranchReference(), null)
-        }
-        val branch = PBranch(version.getTree(), client.getIdGenerator())
-        return branch
-    }
-
     private fun getIncludedModules(root: INode): Iterable<INode> {
         val nameRole = BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name
 

diff --git a/...l-sync-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/GetRevisionInfo.kt b/...l-sync-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/GetRevisionInfo.kt
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.model.sync.bulk.gradle.tasks
+
+import kotlinx.coroutines.runBlocking
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.Optional
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.TaskAction
+import org.modelix.model.client2.ModelClientV2
+import org.modelix.model.lazy.RepositoryId
+
+/**
+ * Determines which revision needs to be pulled from the model server.
+ * The determined revision is written to [revisionFile].
+ */
+abstract class GetRevisionInfo : DefaultTask() {
+
+    @get:Input
+    abstract val serverUrl: Property<String>
+
+    @get:Input
+    abstract val repositoryId: Property<String>
+
+    @get:Input
+    abstract val branchName: Property<String>
+
+    @get:Input
+    @get:Optional
+    abstract val revision: Property<String>
+
+    @get:OutputFile
+    abstract val revisionFile: RegularFileProperty
+
+    @TaskAction
+    fun getRevisionInfo() {
+        val revision = determineRevision()
+        logger.info("Revision to be synced: $revision")
+        revisionFile.get().asFile.writeText(revision)
+    }
+
+    private fun determineRevision(): String {
+        if (revision.isPresent) {
+            return revision.get()
+        }
+
+        logger.info("Pulling versionHash from server...")
+        val modelClient = ModelClientV2.builder().url(serverUrl.get()).build()
+        val branchRef = RepositoryId(repositoryId.get()).getBranchReference(branchName.get())
+
+        return runBlocking {
+            modelClient.use {
+                it.init()
+                it.pullHash(branchRef)
+            }
+        }
+    }
+}
diff --git a/...-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ImportIntoModelServer.kt b/...-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ImportIntoModelServer.kt
@@ -19,7 +19,6 @@ package org.modelix.model.sync.bulk.gradle.tasks
 import kotlinx.coroutines.runBlocking
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.DirectoryProperty
-import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.MapProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.provider.SetProperty
@@ -39,38 +38,37 @@ import org.modelix.model.operations.OTBranch
 import org.modelix.model.sync.bulk.ModelImporter
 import org.modelix.model.sync.bulk.importFilesAsRootChildren
 import org.modelix.model.sync.bulk.isModuleIncluded
-import javax.inject.Inject
 import kotlin.time.Duration.Companion.seconds
 
-abstract class ImportIntoModelServer @Inject constructor(of: ObjectFactory) : DefaultTask() {
+abstract class ImportIntoModelServer : DefaultTask() {
 
-    @InputDirectory
-    @PathSensitive(PathSensitivity.RELATIVE)
-    val inputDir: DirectoryProperty = of.directoryProperty()
+    @get:InputDirectory
+    @get:PathSensitive(PathSensitivity.RELATIVE)
+    abstract val inputDir: DirectoryProperty
 
-    @Input
-    val repositoryId: Property<String> = of.property(String::class.java)
+    @get:Input
+    abstract val repositoryId: Property<String>
 
-    @Input
-    val branchName: Property<String> = of.property(String::class.java)
+    @get:Input
+    abstract val branchName: Property<String>
 
-    @Input
-    val url: Property<String> = of.property(String::class.java)
+    @get:Input
+    abstract val url: Property<String>
 
-    @Input
-    val includedModules: SetProperty<String> = of.setProperty(String::class.java)
+    @get:Input
+    abstract val includedModules: SetProperty<String>
 
-    @Input
-    val includedModulePrefixes: SetProperty<String> = of.setProperty(String::class.java)
+    @get:Input
+    abstract val includedModulePrefixes: SetProperty<String>
 
-    @Input
-    val continueOnError: Property<Boolean> = of.property(Boolean::class.java)
+    @get:Input
+    abstract val continueOnError: Property<Boolean>
 
-    @Input
-    val requestTimeoutSeconds: Property<Int> = of.property(Int::class.java)
+    @get:Input
+    abstract val requestTimeoutSeconds: Property<Int>
 
-    @Input
-    val metaProperties: MapProperty<String, String> = of.mapProperty(String::class.java, String::class.java)
+    @get:Input
+    abstract val metaProperties: MapProperty<String, String>
 
     @TaskAction
     fun import() = runBlocking {

diff --git a/...c-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ValidateSyncSettings.kt b/...c-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ValidateSyncSettings.kt
@@ -17,7 +17,6 @@
 package org.modelix.model.sync.bulk.gradle.tasks
 
 import org.gradle.api.DefaultTask
-import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.CacheableTask
 import org.gradle.api.tasks.Input
@@ -28,18 +27,17 @@ import org.modelix.model.sync.bulk.gradle.config.ModelSyncGradleSettings
 import org.modelix.model.sync.bulk.gradle.config.ServerSource
 import org.modelix.model.sync.bulk.gradle.config.ServerTarget
 import org.modelix.model.sync.bulk.gradle.config.SyncDirection
-import javax.inject.Inject
 
 /**
  * Instead of throwing exceptions for single configuration errors,
  * this task collects all configuration errors and puts them into a single exception,
  * so the user can see all steps that must be taken at a glance.
  */
 @CacheableTask
-abstract class ValidateSyncSettings @Inject constructor(of: ObjectFactory) : DefaultTask() {
+abstract class ValidateSyncSettings : DefaultTask() {
 
-    @Input
-    val settings: Property<ModelSyncGradleSettings> = of.property(ModelSyncGradleSettings::class.java)
+    @get:Input
+    abstract val settings: Property<ModelSyncGradleSettings>
 
     private val errorMsgBuilder = StringBuilder()
 

diff --git a/docs/global/modules/core/pages/reference/component-bulk-model-sync-gradle.adoc b/docs/global/modules/core/pages/reference/component-bulk-model-sync-gradle.adoc
@@ -115,6 +115,10 @@ mpsBuild {
 |===
 
 === ServerSource/-Target configuration
+[WARNING]
+--
+In the future you will be required to specify a `repositoryId` along the`revision`.
+--
 [%header, cols="1,1,2"]
 |===
 |setting
@@ -153,6 +157,7 @@ Only available in ServerTarget.
 
 |===
 
+
 == Example
 
 [source,kotlin]