diff --git a/conf/evolutions/default/1.sql b/conf/evolutions/default/1.sql index a20763e93..4f1886867 100644 --- a/conf/evolutions/default/1.sql +++ b/conf/evolutions/default/1.sql @@ -165,6 +165,7 @@ CREATE TABLE import_url_set_config ( repo_id VARCHAR(50) NOT NULL, import_dataset_id VARCHAR(50) NOT NULL, url_map JSONB NOT NULL, + method VARCHAR(10) NOT NULL DEFAULT 'GET', headers JSONB, created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, comments TEXT, diff --git a/etc/db_migrations/20240129_add_import_url_set_config_method_column.sql b/etc/db_migrations/20240129_add_import_url_set_config_method_column.sql new file mode 100644 index 000000000..bd6cba5f5 --- /dev/null +++ b/etc/db_migrations/20240129_add_import_url_set_config_method_column.sql @@ -0,0 +1,2 @@ +ALTER TABLE import_url_set_config ADD COLUMN method VARCHAR(10) NOT NULL DEFAULT 'GET'; + diff --git a/modules/admin/app/actors/harvesting/UrlSetHarvester.scala b/modules/admin/app/actors/harvesting/UrlSetHarvester.scala index 324fc8fdb..f4f98a6a4 100644 --- a/modules/admin/app/actors/harvesting/UrlSetHarvester.scala +++ b/modules/admin/app/actors/harvesting/UrlSetHarvester.scala @@ -91,8 +91,9 @@ case class UrlSetHarvester (client: WSClient, storage: FileStorage)( val name = item.name val path = job.data.prefix + name - val withHeaders = job.data.config.headers.fold(client.url(item.url)) { headers => - client.url(item.url).withHttpHeaders(headers: _*) + val withMethod = client.url(item.url).withMethod(job.data.config.method) + val withHeaders = job.data.config.headers.fold(withMethod) { headers => + withMethod.withHttpHeaders(headers: _*) } val req = job.data.config.auth.fold(withHeaders) { case BasicAuthConfig(username, password) => withHeaders.withAuth(username, password, WSAuthScheme.BASIC) @@ -120,7 +121,7 @@ case class UrlSetHarvester (client: WSClient, storage: FileStorage)( // Either the hash doesn't match or the file's not there yet // so upload it now... case _ => - val bytes: Future[Source[ByteString, _]] = req.get().map(r => r.bodyAsSource) + val bytes: Future[Source[ByteString, _]] = req.execute().map(r => r.bodyAsSource) bytes.flatMap { src => storage.putBytes( path, diff --git a/modules/admin/app/assets/js/datasets/components/_form-http-headers.vue b/modules/admin/app/assets/js/datasets/components/_form-http-headers.vue index aead28836..9e7016ef9 100644 --- a/modules/admin/app/assets/js/datasets/components/_form-http-headers.vue +++ b/modules/admin/app/assets/js/datasets/components/_form-http-headers.vue @@ -46,12 +46,12 @@ export default {