From 5e81b612d3c64dfeb3eecd3976f3fc3dfa3d2e43 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Fri, 17 Aug 2018 09:49:57 +0300 Subject: [PATCH 01/16] Print url,method and parsed-response --- src/milia/api/http.cljc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/milia/api/http.cljc b/src/milia/api/http.cljc index 72156a2b..1bd2eea7 100644 --- a/src/milia/api/http.cljc +++ b/src/milia/api/http.cljc @@ -35,14 +35,19 @@ [method url & {:keys [accept-header callback filename http-options suppress-4xx-exceptions? raw-response? as-map? no-cache? must-revalidate? auth-token]}] - ;; CLJ: synchronous implementation, checks status before returning. + ;; CLJ: synchronous implementation, checks status before returning. #?(:clj - (let [{:keys [body status] :as response} (http-request + (let [_ (prn ">>>url:" url) + _ (prn ">>>method:" method) + _ (prn ">>>http-options:" http-options) + _ (prn ">>>filename:" filename) + {:keys [body status] :as response} (http-request method url http-options) parsed-response (parse-response body status filename raw-response?) + _ (prn ">>>parsed-response:" parsed-response) error-fn #(throw-error % status parsed-response {:method method :url url :http-options http-options})] From bc233bc64e5c338102df2cf5ab3359ed6b91c586 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Fri, 17 Aug 2018 09:53:53 +0300 Subject: [PATCH 02/16] Check get request status --- src/milia/api/io.clj | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index e91f2c02..e4ef1c03 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -89,7 +89,9 @@ [body filename] (let [tempfile (File/createTempFile filename "") path (str (.getAbsolutePath tempfile)) - ^File file (clojure.java.io/file path)] + ^File file (clojure.java.io/file path) + _ (prn ">>>path:" path) + ] (.deleteOnExit file) ;; Broken out so we can add type hints to avoid reflection (if (instance? String body) @@ -106,6 +108,10 @@ (defn parse-response "Parse a response based on status, filename, and flags" [body status filename raw-response?] + ; {:pre [body]} + (prn ">>>status:" status) + (prn ">>>filename:" filename) + (prn ">>>raw-response?:" raw-response?) (if (and filename (not (error-status? status))) (parse-binary-response body filename) (if raw-response? body (parse-json-response body)))) From a1395426db7680625d91956d246312f49e8f7701 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Fri, 17 Aug 2018 09:55:25 +0300 Subject: [PATCH 03/16] Create testing snapshot --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4ec1ad2f..4193e55f 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,7 @@ :milia-http-default-per-route "10" :milia-http-threads "20"}) -(defproject onaio/milia "0.3.40-SNAPSHOT" +(defproject onaio/milia "0.3.41-SNAPSHOT-DEBUG-API" :description "The ona.io Clojure Web API Client." :dependencies [;; CORE MILIA REQUIREMENTS [cheshire "5.6.3"] From e912d4d0666846eb3445b9b329d48eb6423c7173 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Fri, 17 Aug 2018 15:44:02 +0300 Subject: [PATCH 04/16] Add check for filename before printing --- src/milia/api/http.cljc | 10 +++++----- src/milia/api/io.clj | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/milia/api/http.cljc b/src/milia/api/http.cljc index 1bd2eea7..823691ed 100644 --- a/src/milia/api/http.cljc +++ b/src/milia/api/http.cljc @@ -37,17 +37,17 @@ no-cache? must-revalidate? auth-token]}] ;; CLJ: synchronous implementation, checks status before returning. #?(:clj - (let [_ (prn ">>>url:" url) - _ (prn ">>>method:" method) - _ (prn ">>>http-options:" http-options) - _ (prn ">>>filename:" filename) + (let [_ (when filename (prn ">>>url:" url)) + _ (when filename (prn ">>>method:" method)) + _ (when filename (prn ">>>http-options:" http-options)) + _ (when filename (prn ">>>filename:" filename)) {:keys [body status] :as response} (http-request method url http-options) parsed-response (parse-response body status filename raw-response?) - _ (prn ">>>parsed-response:" parsed-response) + _ (when filename (prn ">>>parsed-response:" parsed-response)) error-fn #(throw-error % status parsed-response {:method method :url url :http-options http-options})] diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index e4ef1c03..bbb259e9 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -90,7 +90,7 @@ (let [tempfile (File/createTempFile filename "") path (str (.getAbsolutePath tempfile)) ^File file (clojure.java.io/file path) - _ (prn ">>>path:" path) + _ (when filename (prn ">>>path:" path)) ] (.deleteOnExit file) ;; Broken out so we can add type hints to avoid reflection @@ -109,9 +109,10 @@ "Parse a response based on status, filename, and flags" [body status filename raw-response?] ; {:pre [body]} + (when filename (prn ">>>status:" status) (prn ">>>filename:" filename) - (prn ">>>raw-response?:" raw-response?) + (prn ">>>raw-response?:" raw-response?)) (if (and filename (not (error-status? status))) (parse-binary-response body filename) (if raw-response? body (parse-json-response body)))) From f0c08dc7c56a6b2c85973f52708a9584dce6c86a Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Fri, 17 Aug 2018 16:09:48 +0300 Subject: [PATCH 05/16] Remove filename check --- src/milia/api/http.cljc | 10 +++++----- src/milia/api/io.clj | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/milia/api/http.cljc b/src/milia/api/http.cljc index 823691ed..1bd2eea7 100644 --- a/src/milia/api/http.cljc +++ b/src/milia/api/http.cljc @@ -37,17 +37,17 @@ no-cache? must-revalidate? auth-token]}] ;; CLJ: synchronous implementation, checks status before returning. #?(:clj - (let [_ (when filename (prn ">>>url:" url)) - _ (when filename (prn ">>>method:" method)) - _ (when filename (prn ">>>http-options:" http-options)) - _ (when filename (prn ">>>filename:" filename)) + (let [_ (prn ">>>url:" url) + _ (prn ">>>method:" method) + _ (prn ">>>http-options:" http-options) + _ (prn ">>>filename:" filename) {:keys [body status] :as response} (http-request method url http-options) parsed-response (parse-response body status filename raw-response?) - _ (when filename (prn ">>>parsed-response:" parsed-response)) + _ (prn ">>>parsed-response:" parsed-response) error-fn #(throw-error % status parsed-response {:method method :url url :http-options http-options})] diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index bbb259e9..e4ef1c03 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -90,7 +90,7 @@ (let [tempfile (File/createTempFile filename "") path (str (.getAbsolutePath tempfile)) ^File file (clojure.java.io/file path) - _ (when filename (prn ">>>path:" path)) + _ (prn ">>>path:" path) ] (.deleteOnExit file) ;; Broken out so we can add type hints to avoid reflection @@ -109,10 +109,9 @@ "Parse a response based on status, filename, and flags" [body status filename raw-response?] ; {:pre [body]} - (when filename (prn ">>>status:" status) (prn ">>>filename:" filename) - (prn ">>>raw-response?:" raw-response?)) + (prn ">>>raw-response?:" raw-response?) (if (and filename (not (error-status? status))) (parse-binary-response body filename) (if raw-response? body (parse-json-response body)))) From 9115555dfb6dbb007fef61552b98a5e438cb204c Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Mon, 27 Aug 2018 15:12:14 +0300 Subject: [PATCH 06/16] Add function for check file extension --- src/milia/utils/file.clj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/milia/utils/file.clj b/src/milia/utils/file.clj index 4b7c1acc..a18ae926 100644 --- a/src/milia/utils/file.clj +++ b/src/milia/utils/file.clj @@ -2,6 +2,8 @@ (:require [clojure.java.io :as io]) (:import [org.apache.commons.io IOUtils])) +(def file-extension-regex #"\w+$") + (defn to-byte-array [data-file] (IOUtils/toByteArray (io/input-stream data-file))) @@ -16,3 +18,9 @@ (.deleteOnExit tempdir) (clojure.java.io/copy tempfile file) file)) + +(defn get-file-extension + "Get the file extension given full file name. + String -> String" + [filename] + (re-find file-extension-regex filename)) \ No newline at end of file From e2db521affef4439c998b87331e150e84c191279 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Mon, 27 Aug 2018 17:20:37 +0300 Subject: [PATCH 07/16] Call parse-binary-response function if filename has a .json extension which will handle the http request --- src/milia/api/http.cljc | 50 +++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/src/milia/api/http.cljc b/src/milia/api/http.cljc index 1bd2eea7..88b2faa0 100644 --- a/src/milia/api/http.cljc +++ b/src/milia/api/http.cljc @@ -1,7 +1,9 @@ (ns milia.api.http (:require [clojure.set :refer [rename-keys]] #?@(:clj [[milia.api.io :refer [build-req parse-response - http-request debug-api]] + http-request debug-api + parse-binary-response]] + [milia.utils.file :as file-utils] [slingshot.slingshot :refer [throw+]]] :cljs [[milia.api.io :refer [build-http-options token->headers http-request raw-request]] @@ -35,31 +37,41 @@ [method url & {:keys [accept-header callback filename http-options suppress-4xx-exceptions? raw-response? as-map? no-cache? must-revalidate? auth-token]}] - ;; CLJ: synchronous implementation, checks status before returning. + ;; CLJ: synchronous implementation, checks status before returning. #?(:clj - (let [_ (prn ">>>url:" url) - _ (prn ">>>method:" method) - _ (prn ">>>http-options:" http-options) - _ (prn ">>>filename:" filename) - {:keys [body status] :as response} (http-request - method url http-options) - parsed-response (parse-response body - status - filename - raw-response?) - _ (prn ">>>parsed-response:" parsed-response) + (let [json-file? + (when (and filename (not raw-response?)) + (= "json" (file-utils/get-file-extension filename))) + ;; Call http-request if filename extension is not of type json + {:keys [body status] :as response} + (when-not json-file? + (http-request method + url + http-options)) + ;; Call parse-binary-response if filename extension is of type json + ;; which will then handle the http request + parsed-response (if json-file? + (parse-binary-response nil + filename + :url url + :http-options http-options) + (parse-response body + status + filename + raw-response?)) error-fn #(throw-error % status parsed-response {:method method :url url :http-options http-options})] (debug-api method url http-options response) ;; Assume that a nil status indicates an exception - (cond - (nil? response) (error-fn :no-http-response) - (nil? status) (error-fn :no-http-status) - (and status + (when-not json-file? + (cond + (nil? response) (error-fn :no-http-response) + (nil? status) (error-fn :no-http-status) + (and status (>= status 400) (< status 500) (not suppress-4xx-exceptions?)) - (error-fn :http-client-error) - (>= status 500) (error-fn :http-server-error)) + (error-fn :http-client-error) + (>= status 500) (error-fn :http-server-error))) (if as-map? (assoc response :body parsed-response) parsed-response)) ;; CLJS: asynchronous implementation, returns a channel. From 83b8d8e2a9bec9d0ded32be65b27da2d1a1f3568 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Mon, 27 Aug 2018 17:22:09 +0300 Subject: [PATCH 08/16] Stream the http request to avoid out of memory errors then use java io/copy method to copy the data to file --- src/milia/api/io.clj | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index e4ef1c03..1bfa3d88 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -86,32 +86,41 @@ (defn parse-binary-response "Parse binary response by writing into a temp file and returning the path." - [body filename] + [body filename & {:keys [url http-options]}] (let [tempfile (File/createTempFile filename "") path (str (.getAbsolutePath tempfile)) ^File file (clojure.java.io/file path) - _ (prn ">>>path:" path) - ] + ;; Stream the http-request to avoid out of memory errors when the data + ;; to copy is large + streamed-http-request + (client/get url (build-req (assoc http-options :as (keyword "stream")))) + json-file? + (when filename (= "json" (file-utils/get-file-extension filename)))] (.deleteOnExit file) - ;; Broken out so we can add type hints to avoid reflection - (if (instance? String body) - (let [^String body-string body] - (with-open [^java.io.Writer w (io/writer file :append false)] - (.write w body-string))) - (let [^bytes body-bytes body] - (with-open [^java.io.OutputStream w (io/output-stream file - :append - false)] - (.write w body-bytes)))) + ;; io/copy is used since it takes an input-stream and an output-stream + (if (and json-file? (not (error-status? (:status streamed-http-request)))) + (with-open [in-stream (:body streamed-http-request) + out-stream (->> file + io/as-file + io/output-stream)] + (io/copy in-stream out-stream)) + (parse-json-response (:body streamed-http-request))) + ;; Broken out so we can add type hints to avoid reflection + (when-not json-file? + (if (instance? String body) + (let [^String body-string body] + (with-open [^java.io.Writer w (io/writer file :append false)] + (.write w body-string))) + (let [^bytes body-bytes body] + (with-open [^java.io.OutputStream w (io/output-stream file + :append + false)] + (.write w body-bytes))))) path)) (defn parse-response "Parse a response based on status, filename, and flags" [body status filename raw-response?] - ; {:pre [body]} - (prn ">>>status:" status) - (prn ">>>filename:" filename) - (prn ">>>raw-response?:" raw-response?) (if (and filename (not (error-status? status))) (parse-binary-response body filename) (if raw-response? body (parse-json-response body)))) From 1d3b459b6aef42eca84823126dfb686246fb1dcd Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Mon, 27 Aug 2018 17:23:18 +0300 Subject: [PATCH 09/16] Add test for the json extension file type logic --- test/clj/milia/api/io_test.clj | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/clj/milia/api/io_test.clj b/test/clj/milia/api/io_test.clj index cb31ef72..08605620 100644 --- a/test/clj/milia/api/io_test.clj +++ b/test/clj/milia/api/io_test.clj @@ -93,7 +93,17 @@ url http-options) => response - (parse-binary-response body :filename) => file)))) + (parse-binary-response body :filename) => file))) + (fact "should return a file when filename has a .json extension" + (let [http-options {:multipart []}] + (parse-http :method url + :http-options http-options + :filename "filename.json") => (contains file) + (provided + (parse-binary-response nil + "filename.json" + :url url + :http-options http-options) => file)))) (facts "about http-request" (fact "should add digest if account has password" From 46772a0a502308e2a5ec0607a14dea8919662434 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Mon, 27 Aug 2018 17:25:05 +0300 Subject: [PATCH 10/16] Rename milia snapshot --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4193e55f..bb03bd15 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,7 @@ :milia-http-default-per-route "10" :milia-http-threads "20"}) -(defproject onaio/milia "0.3.41-SNAPSHOT-DEBUG-API" +(defproject onaio/milia "0.3.41-SNAPSHOT" :description "The ona.io Clojure Web API Client." :dependencies [;; CORE MILIA REQUIREMENTS [cheshire "5.6.3"] From dfa3b534eda8f8238a9220c445e2ea791c7e28b9 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Tue, 28 Aug 2018 15:50:58 +0300 Subject: [PATCH 11/16] Remove file-extension logic --- src/milia/utils/file.clj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/milia/utils/file.clj b/src/milia/utils/file.clj index a18ae926..4b7c1acc 100644 --- a/src/milia/utils/file.clj +++ b/src/milia/utils/file.clj @@ -2,8 +2,6 @@ (:require [clojure.java.io :as io]) (:import [org.apache.commons.io IOUtils])) -(def file-extension-regex #"\w+$") - (defn to-byte-array [data-file] (IOUtils/toByteArray (io/input-stream data-file))) @@ -18,9 +16,3 @@ (.deleteOnExit tempdir) (clojure.java.io/copy tempfile file) file)) - -(defn get-file-extension - "Get the file extension given full file name. - String -> String" - [filename] - (re-find file-extension-regex filename)) \ No newline at end of file From c4aaf010c775552169745acf17ce61ed02f29e11 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Tue, 28 Aug 2018 15:51:50 +0300 Subject: [PATCH 12/16] Check filename extension using .endsWith method --- src/milia/api/http.cljc | 3 +-- src/milia/api/io.clj | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/milia/api/http.cljc b/src/milia/api/http.cljc index 88b2faa0..e45581cc 100644 --- a/src/milia/api/http.cljc +++ b/src/milia/api/http.cljc @@ -3,7 +3,6 @@ #?@(:clj [[milia.api.io :refer [build-req parse-response http-request debug-api parse-binary-response]] - [milia.utils.file :as file-utils] [slingshot.slingshot :refer [throw+]]] :cljs [[milia.api.io :refer [build-http-options token->headers http-request raw-request]] @@ -41,7 +40,7 @@ #?(:clj (let [json-file? (when (and filename (not raw-response?)) - (= "json" (file-utils/get-file-extension filename))) + (.endsWith filename ".json")) ;; Call http-request if filename extension is not of type json {:keys [body status] :as response} (when-not json-file? diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index 1bfa3d88..93014942 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -95,7 +95,7 @@ streamed-http-request (client/get url (build-req (assoc http-options :as (keyword "stream")))) json-file? - (when filename (= "json" (file-utils/get-file-extension filename)))] + (when filename (.endsWith filename ".json"))] (.deleteOnExit file) ;; io/copy is used since it takes an input-stream and an output-stream (if (and json-file? (not (error-status? (:status streamed-http-request)))) From 0557efea8d0776d0c7bd6e626805bc92147f32d0 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Tue, 28 Aug 2018 16:31:58 +0300 Subject: [PATCH 13/16] Destructure body and status keys from http request --- src/milia/api/io.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index 93014942..d1d5478c 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -92,19 +92,19 @@ ^File file (clojure.java.io/file path) ;; Stream the http-request to avoid out of memory errors when the data ;; to copy is large - streamed-http-request + {streamed-body :body status :status} (client/get url (build-req (assoc http-options :as (keyword "stream")))) json-file? (when filename (.endsWith filename ".json"))] (.deleteOnExit file) ;; io/copy is used since it takes an input-stream and an output-stream - (if (and json-file? (not (error-status? (:status streamed-http-request)))) - (with-open [in-stream (:body streamed-http-request) + (if (and json-file? (not (error-status? status))) + (with-open [in-stream streamed-body out-stream (->> file io/as-file io/output-stream)] (io/copy in-stream out-stream)) - (parse-json-response (:body streamed-http-request))) + (parse-json-response streamed-body)) ;; Broken out so we can add type hints to avoid reflection (when-not json-file? (if (instance? String body) From 262fecdd2070393c6f7f56def7a178977d21dc97 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Tue, 28 Aug 2018 21:59:01 +0300 Subject: [PATCH 14/16] Refactor code --- src/milia/api/io.clj | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index d1d5478c..83dc93af 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -99,11 +99,10 @@ (.deleteOnExit file) ;; io/copy is used since it takes an input-stream and an output-stream (if (and json-file? (not (error-status? status))) - (with-open [in-stream streamed-body - out-stream (->> file + (with-open [out-stream (->> file io/as-file io/output-stream)] - (io/copy in-stream out-stream)) + (io/copy streamed-body out-stream)) (parse-json-response streamed-body)) ;; Broken out so we can add type hints to avoid reflection (when-not json-file? From d406df391f01d365ac90deb111a19d8aa917d2d9 Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Wed, 29 Aug 2018 08:31:15 +0300 Subject: [PATCH 15/16] Out dent comment --- src/milia/api/io.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/milia/api/io.clj b/src/milia/api/io.clj index 83dc93af..82568a73 100644 --- a/src/milia/api/io.clj +++ b/src/milia/api/io.clj @@ -104,7 +104,7 @@ io/output-stream)] (io/copy streamed-body out-stream)) (parse-json-response streamed-body)) - ;; Broken out so we can add type hints to avoid reflection + ;; Broken out so we can add type hints to avoid reflection (when-not json-file? (if (instance? String body) (let [^String body-string body] From c152262d65e29e5caec4f9e1154590877b025a4e Mon Sep 17 00:00:00 2001 From: John Mwashuma Date: Thu, 30 Aug 2018 09:59:40 +0300 Subject: [PATCH 16/16] Update milia version to 0.3.41 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index bb03bd15..983ef5f5 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,7 @@ :milia-http-default-per-route "10" :milia-http-threads "20"}) -(defproject onaio/milia "0.3.41-SNAPSHOT" +(defproject onaio/milia "0.3.41" :description "The ona.io Clojure Web API Client." :dependencies [;; CORE MILIA REQUIREMENTS [cheshire "5.6.3"]