Skip to content

Commit

Permalink
🎉 Add binfile-v3 export/import file format
Browse files Browse the repository at this point in the history
  • Loading branch information
@niwinz
niwinz committed Oct 18, 2024
1 parent 4fb5d3f commit 8618cb9
Show file tree
Hide file tree
Showing 35 changed files with 2,039 additions and 607 deletions.
77 changes: 68 additions & 9 deletions backend/src/app/binfile/common.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,21 @@
(def ^:dynamic *state* nil)
(def ^:dynamic *options* nil)

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; DEFAULTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; Threshold in MiB when we pass from using
;; in-memory byte-array's to use temporal files.
(def temp-file-threshold
(* 1024 1024 2))

;; A maximum (storage) object size allowed: 100MiB
(def ^:const max-object-size
(* 1024 1024 100))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(def xf-map-id
(map :id))

Expand All @@ -56,6 +71,13 @@
(def conj-vec
(fnil conj []))

(defn initial-state
[]
{:storage-objects #{}
:files #{}
:teams #{}
:projects #{}})

(defn collect-storage-objects
[state items]
(update state :storage-objects into xf-map-media-id items))
Expand Down Expand Up @@ -87,6 +109,8 @@
attrs))

(defn update-index
([coll]
(update-index {} coll identity))
([index coll]
(update-index index coll identity))
([index coll attr]
Expand Down Expand Up @@ -114,6 +138,16 @@
[cfg project-id]
(db/get cfg :project {:id project-id}))

(def ^:private sql:get-teams
"SELECT t.* FROM team WHERE id = ANY(?)")

(defn get-teams
[cfg ids]
(let [conn (db/get-connection cfg)
ids (db/create-array conn "uuid" ids)]
(->> (db/exec! conn [sql:get-teams ids])
(map decode-row))))

(defn get-team
[cfg team-id]
(-> (db/get cfg :team {:id team-id})
Expand Down Expand Up @@ -167,9 +201,10 @@
(defn get-file-object-thumbnails
"Return all file object thumbnails for a given file."
[cfg file-id]
(db/query cfg :file-tagged-object-thumbnail
{:file-id file-id
:deleted-at nil}))
(->> (db/query cfg :file-tagged-object-thumbnail
{:file-id file-id
:deleted-at nil})
(not-empty)))

(defn get-file-thumbnail
"Return the thumbnail for the specified file-id"
Expand Down Expand Up @@ -224,26 +259,26 @@
(->> (db/exec! conn [sql ids])
(mapv #(assoc % :file-id id)))))))

(def ^:private sql:get-team-files
(def ^:private sql:get-team-files-ids
"SELECT f.id FROM file AS f
JOIN project AS p ON (p.id = f.project_id)
WHERE p.team_id = ?")

(defn get-team-files
(defn get-team-files-ids
"Get a set of file ids for the specified team-id"
[{:keys [::db/conn]} team-id]
(->> (db/exec! conn [sql:get-team-files team-id])
(->> (db/exec! conn [sql:get-team-files-ids team-id])
(into #{} xf-map-id)))

(def ^:private sql:get-team-projects
"SELECT p.id FROM project AS p
"SELECT p.* FROM project AS p
WHERE p.team_id = ?
AND p.deleted_at IS NULL")

(defn get-team-projects
"Get a set of project ids for the team"
[{:keys [::db/conn]} team-id]
(->> (db/exec! conn [sql:get-team-projects team-id])
[cfg team-id]
(->> (db/exec! cfg [sql:get-team-projects team-id])
(into #{} xf-map-id)))

(def ^:private sql:get-project-files
Expand All @@ -257,6 +292,10 @@
(->> (db/exec! conn [sql:get-project-files project-id])
(into #{} xf-map-id)))

(defn remap-thumbnail-object-id
[object-id file-id]
(str/replace-first object-id #"^(.*?)/" (str file-id "/")))

(defn- relink-shapes
"A function responsible to analyze all file data and
replace the old :component-file reference with the new
Expand Down Expand Up @@ -339,6 +378,12 @@
data
library-ids)))

(defn disable-database-timeouts!
[cfg]
(let [conn (db/get-connection cfg)]
(db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))

(defn- fix-version
[file]
(let [file (fmg/fix-version file)]
Expand Down Expand Up @@ -432,6 +477,20 @@

file))


(defn register-pending-migrations
"All features that are enabled and requires explicit migration are
added to the state for a posterior migration step."
[cfg {:keys [id features] :as file}]
(doseq [feature (-> (::features cfg)
(set/difference cfeat/no-migration-features)
(set/difference cfeat/backend-only-features)
(set/difference features))]
(vswap! *state* update :pending-to-migrate (fnil conj []) [feature id]))

file)


(defn apply-pending-migrations!
"Apply alredy registered pending migrations to files"
[cfg]
Expand Down
34 changes: 12 additions & 22 deletions backend/src/app/binfile/v1.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,27 +49,13 @@

(set! *warn-on-reflection* true)

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; DEFAULTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; Threshold in MiB when we pass from using
;; in-memory byte-array's to use temporal files.
(def temp-file-threshold
(* 1024 1024 2))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; LOW LEVEL STREAM IO API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(def ^:const buffer-size (:xnio/buffer-size yt/defaults))
(def ^:const penpot-magic-number 800099563638710213)


;; A maximum (storage) object size allowed: 100MiB
(def ^:const max-object-size
(* 1024 1024 100))

(def ^:dynamic *position* nil)

(defn get-mark
Expand Down Expand Up @@ -258,12 +244,12 @@
p (tmp/tempfile :prefix "penpot.binfile.")]
(assert-mark m :stream)

(when (> s max-object-size)
(when (> s bfc/max-object-size)
(ex/raise :type :validation
:code :max-file-size-reached
:hint (str/ffmt "unable to import storage object with size % bytes" s)))

(if (> s temp-file-threshold)
(if (> s bfc/temp-file-threshold)
(with-open [^OutputStream output (io/output-stream p)]
(let [readed (io/copy! input output :offset 0 :size s)]
(l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/sync? true)
Expand Down Expand Up @@ -381,10 +367,12 @@
::l/sync? true)

(doseq [item media]
(l/dbg :hint "write penpot file media object" :id (:id item) ::l/sync? true))
(l/dbg :hint "write penpot file media object"
:id (:id item) ::l/sync? true))

(doseq [item thumbnails]
(l/dbg :hint "write penpot file object thumbnail" :media-id (str (:media-id item)) ::l/sync? true))
(l/dbg :hint "write penpot file object thumbnail"
:media-id (str (:media-id item)) ::l/sync? true))

(doto output
(write-obj! file)
Expand Down Expand Up @@ -466,8 +454,8 @@

(defn- read-import-v1
[{:keys [::db/conn ::project-id ::profile-id ::input] :as cfg}]
(db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])

(bfc/disable-database-timeouts! cfg)

(pu/with-open [input (zstd-input-stream input)
input (io/data-input-stream input)]
Expand Down Expand Up @@ -559,7 +547,9 @@

(when (seq thumbnails)
(let [thumbnails (remap-thumbnails thumbnails file-id')]
(l/dbg :hint "updated index with thumbnails" :total (count thumbnails) ::l/sync? true)
(l/dbg :hint "updated index with thumbnails"
:total (count thumbnails)
::l/sync? true)
(vswap! bfc/*state* update :thumbnails bfc/into-vec thumbnails)))

(when (seq media)
Expand Down Expand Up @@ -738,7 +728,7 @@
:cause @cs)))))

(defn import-files!
[cfg input]
[{:keys [::input] :as cfg}]

(dm/assert!
"expected valid profile-id and project-id on `cfg`"
Expand Down
21 changes: 10 additions & 11 deletions backend/src/app/binfile/v2.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,16 +141,15 @@
(write! cfg :team-font-variant id font))))

(defn- write-project!
[cfg project-id]
(let [project (bfc/get-project cfg project-id)]
(events/tap :progress
{:op :export
:section :write-project
:id project-id
:name (:name project)})
(l/trc :hint "write" :obj "project" :id (str project-id))
(write! cfg :project (str project-id) project)
(vswap! bfc/*state* update :projects conj project-id)))
[cfg project]
(events/tap :progress
{:op :export
:section :write-project
:id (:id project)
:name (:name project)})
(l/trc :hint "write" :obj "project" :id (str (:id project)))
(write! cfg :project (str (:id project)) project)
(vswap! bfc/*state* update :projects conj (:id project)))

(defn- write-file!
[cfg file-id]
Expand Down Expand Up @@ -363,7 +362,7 @@
(bfc/get-team-projects cfg team-id))

(run! (partial write-file! cfg)
(bfc/get-team-files cfg team-id))
(bfc/get-team-files-ids cfg team-id))

(run! (partial write-storage-object! cfg)
(-> bfc/*state* deref :storage-objects))
Expand Down
Loading

0 comments on commit 8618cb9

Please sign in to comment.