;;;; fussy.lisp (in-package #:fussy) (defvar +default-emacs-package-archive+ "https://melpa.org/packages/archive-contents") (defparameter +retry-wait+ 60 "Nubmer of seconds to wait between http request retries.") (defun emacs-reader-readtable () "Return a readtable that will read the emacs package archive contents." (let ((*readtable* (copy-readtable nil))) (labels ((bracket-reader (stream char) (declare (ignorable char)) (read-delimited-list #\] stream))) (set-macro-character #\[ #'bracket-reader) (set-macro-character #\] (get-macro-character #\) nil)) *readtable*))) (defun fetch-emacs-archive (&optional (archive +default-emacs-package-archive+) (retries 10)) "Fetch the package archive from ARCHIVE, a url, and read it in using the emacs' reader readtable." (when (plusp retries) (fussy-log "Fetching archive ~a~%" archive) (handler-case (multiple-value-bind (stream status) (dexador:get archive :want-stream t) (when (= 200 status) (let* ((*package* (find-package :fussy)) (*readtable* (emacs-reader-readtable))) (read stream)))) (usocket:ns-host-not-found-error () (fussy-log "Error whild fetching archive.~%Host Not Found: ~a~%Retrying in ~a seconds...~%" archive +retry-wait+) (sleep +retry-wait+) (fetch-emacs-archive archive (1- retries))) (error (e) (fussy-log "Error while fetching archive: ~a~%Retrying in ~a seconds...~%" e +retry-wait+) (sleep +retry-wait+) (fetch-emacs-archive archive (1- retries)))))) (defclass/std theme-pkg (db:store-object) ((name :with :index-type bknr.indices:unique-index :index-reader theme-pkg-with-name) (commit version authors maintainer keywords url description contained-themes :with)) (:metaclass db:persistent-class) (:documentation "Represents an Emacs theme package.")) (defmethod print-object ((theme theme-pkg) stream) (format stream "#" (theme-pkg-name theme) (theme-pkg-version theme))) (defparameter +excludes+ '(tramp-theme color-theme airline-themes unobtrusive-magit-theme cycle-themes dir-treeview-themes display-theme smart-mode-line-powerline-theme smart-mode-line-atom-one-dark-theme rand-theme per-buffer-theme svg-mode-line-themes remember-last-theme xresources-theme)) (defun theme-package-p (archive-object) (when (consp archive-object) (unless (member (first archive-object) +excludes+) (let ((name (symbol-name (first archive-object)))) (or (a:ends-with-subseq "-theme" name :test #'char-equal) (a:ends-with-subseq "-themes" name :test #'char-equal)))))) (defun archive-theme-name (archive-pkg) (first archive-pkg)) (defun archive-theme-version (archive-pkg) (second archive-pkg)) (defun make-theme-pkg-from-archive-theme (archive-theme) (destructuring-bind (name version _dontcare desc _dontcare2 meta-alist) archive-theme (declare (ignore _dontcare _dontcare2)) (db:with-transaction () (make-instance 'theme-pkg :name name :version version :description desc :commit (cdr (assoc :commit meta-alist)) :maintainer (cdr (assoc :maintainer meta-alist)) :url (cdr (assoc :url meta-alist)) :authors (cdr (assoc :authors meta-alist)) :keywords (cdr (assoc :keywords meta-alist)))))) (defun update-theme-pkg-from-archive-theme (pkg arch) (destructuring-bind (name version _dontcare desc _dontcare2 meta-alist) arch (declare (ignore _dontcare _dontcare2)) (db:with-transaction () (setf (theme-pkg-name pkg) name (theme-pkg-version pkg) version (theme-pkg-description pkg) desc (theme-pkg-commit pkg) (cdr (assoc :commit meta-alist)) (theme-pkg-maintainer pkg) (cdr (assoc :maintainer meta-alist)) (theme-pkg-url pkg) (cdr (assoc :url meta-alist)) (theme-pkg-authors pkg) (cdr (assoc :authors meta-alist)) (theme-pkg-keywords pkg) (cdr (assoc :keywords meta-alist)))))) (defun package-name<= (pkg1 pkg2) (string<= (symbol-name (theme-pkg-name pkg1)) (symbol-name (theme-pkg-name pkg2)))) (defun package-version<= (pkg1 pkg2) (<= (first (theme-pkg-version pkg1)) (first (theme-pkg-version pkg2)))) (defun process-archive-theme (archive-theme) ;; ugg this is ugly. I hate it when code is all dense like this. (a:if-let (pkg (theme-pkg-with-name (archive-theme-name archive-theme))) (when (not (equalp (archive-theme-version archive-theme) (theme-pkg-version pkg))) (update-theme-pkg-from-archive-theme pkg archive-theme)) (make-theme-pkg-from-archive-theme archive-theme))) (defun theme-needs-update-p (archive-theme) "A theme should be updated if eithe it is new or if its version differs from the theme-pkg instance already in the data store." (let ((pkg (theme-pkg-with-name (archive-theme-name archive-theme)))) (not (and pkg (equalp (archive-theme-version archive-theme) (theme-pkg-version pkg)))))) (defun update-theme-packages () (let ((themes-to-update (remove-if-not (a:conjoin #'theme-package-p #'theme-needs-update-p) (fetch-emacs-archive)))) ;; just going to let this throw an error. will either introduce a ;; restart-case with some restarts or something later, or will ;; just catch the error in the caller of update-theme-packages (when themes-to-update (fussy-log "Updating Themes") ;; delete the temporary .emacs.d (let ((emacs-dir (uiop:merge-pathnames* ".emacs.d/" (full-theme-image-directory)))) (when (uiop:directory-exists-p emacs-dir) (fussy-log "Deleting emacs dir at: ~a" emacs-dir) (uiop:delete-directory-tree emacs-dir :validate t :if-does-not-exist :ignore) (if (uiop:directory-exists-p emacs-dir) (fussy-log "Directory NOT DELETE!?") (fussy-log "Directory successfully deleted")))) (handler-case (generate-images-for-packages (mapcar #'archive-theme-name themes-to-update)) (error (e) (fussy-log "Error while generating images: ~s" e))) ;; if we didn't error: update the db (dolist (archive-theme themes-to-update) (process-archive-theme archive-theme)) (reindex-themes-by-package) (fussy-log "Done Updating Themes")))) (defun all-theme-keywords () (delete-duplicates (copy-seq (mapcan (a:compose #'copy-seq #'theme-pkg-keywords) (db:store-objects-with-class 'theme-pkg))) :test #'equal)) (defun themes-with-keywords (&rest keywords) (loop :for theme :in (db:store-objects-with-class 'theme-pkg) :when (subsetp keywords (theme-pkg-keywords theme) :test #'string-equal) :collect theme)) (defun all-theme-packages (&optional sort-by) (let ((packages (db:store-objects-with-class 'theme-pkg))) (if sort-by (sort (copy-seq packages) sort-by) packages))) (defun just-the-directory-namestring (path) "returns a string, the name of the directory that the path represents." ;; directory-namestring is taken, seems to peel off any file at the end (first (last (pathname-directory path)))) (defun themes-in-package (pkg &optional (config *config*)) "Themes that belong to a particular package are stored as the names of directories that contain the theme's images. These directories are stored beneath the directory that names the package. This function returns a list of those names." (mapcar #'just-the-directory-namestring (uiop:subdirectories (uiop:merge-pathnames* (string-downcase (format nil "~a/" (theme-pkg-name pkg))) (full-theme-image-directory config))))) (defun all-themes (&key sort-by) (a:mappend #'themes-in-package (all-theme-packages sort-by))) (defvar *theme-matcher-cache* (make-hash-table :test #'equal)) (defun theme-term-matcher (term) (if (eq #\- (elt term 0)) (complement (theme-term-matcher (string-left-trim "-" term))) (lambda (theme) (or (search term theme) (with-slots (name keywords description authors maintainer) (theme-package theme) (or (search term (symbol-name name) :test #'char-equal) (search term description :test #'char-equal))))))) (defun theme-mentions-anywhere (term) (or (gethash (string-downcase term) *theme-matcher-cache*) (setf (gethash (string-downcase term) *theme-matcher-cache*) (theme-term-matcher term)))) (defun search-themes (terms &key sort-by) (remove-if-not (apply #'a:conjoin (mapcar #'theme-mentions-anywhere terms)) (all-themes :sort-by sort-by))) (defvar *theme->packages* nil "A hash table indexing packages by the themes they contain.") (defun theme-package (theme) (gethash theme *theme->packages*)) (defun search-packages (terms &key sort-by) ;; slow but I get to recycle (remove-duplicates (mapcar #'theme-package (search-themes terms :sort-by sort-by)) :test #'eq)) (defun all-packages () (db:store-objects-with-class 'theme-pkg)) (defun reindex-themes-by-package () (let ((table (make-hash-table :test #'equal))) (dolist (pkg (all-theme-packages)) (dolist (theme (themes-in-package pkg)) (setf (gethash theme table) pkg))) (setf *theme->packages* table))) (defgeneric url-path (theme)) (defmethod url-path ((theme theme-pkg)) (format nil "/package/~a" (string-downcase (theme-pkg-name theme)))) (defmethod url-path ((theme string)) (assert (theme-p theme)) (format nil "/theme/~a/~a" (string-downcase (theme-pkg-name (theme-package theme))) theme)) (defun theme-p (string) (not (null (theme-package string))) ) (defclass/std config () ((theme-image-directory store-directory fussy-el :std "" :documentation "These paths are by default equal to the config directory.") (robots.txt :std "robots.txt") (logfile :std "fussy.log") (port :std 8888) (domain :std "localhost") (address :std "0.0.0.0") (fetch-time :std (list 17 18) :documentation "(Hour Minute) at which fussy updates its themes."))) (defvar *config* nil) (defvar *config-directory* nil "The config dir is the directory where the config file is. This directory is used as a default directory for all relative paths mentioned in the config file. Any Absolute paths in the config file are treated as such.") (defun config-directory () (or *config-directory* (uiop:merge-pathnames* (user-homedir-pathname) ".fussy/"))) (defun robots-path-name (&optional (config *config*)) (uiop:merge-pathnames* (config-directory) (robots.txt config))) (defun logfile-path-name (&optional (config *config*)) (uiop:merge-pathnames* (config-directory) (logfile config))) (defun fussy-log (msg &rest args) (with-open-file (out (logfile-path-name) :direction :output :if-exists :append :if-does-not-exist :create) (multiple-value-bind (sec min hour date month year) (get-decoded-time) (format out "~4d/~2,'0d/~2,'0d ~2,'0d:~2,'0d:~2,'0d " year month date hour min sec)) (apply #'format out msg args) (terpri out))) (defun load-config-from-file (file) (apply #'make-instance 'config (read-from-string (a:read-file-into-string file)))) (defun full-store-directory (&optional (config *config*)) (uiop:merge-pathnames* (store-directory config) (config-directory))) (defun full-theme-image-directory (&optional (config *config*)) (uiop:merge-pathnames* (theme-image-directory config) (config-directory))) (defun emacs-dot-d-directory (&optional (config *config*)) (uiop:merge-pathnames* ".emacs.d/" (full-theme-image-directory config))) (defun create-db () (unless (boundp 'db:*store* ) (ensure-directories-exist (full-store-directory)) (make-instance 'db:mp-store :directory (full-store-directory) :subsystems (list (make-instance 'db:store-object-subsystem))))) (defparameter +standard-themes+ '(adwaita deeper-blue dichromacy light-blue tango-dark wheatgrass manoj-dark tsdh-dark tsdh-light whiteboard leuven misterioso tango wombat) "these are built in themes - this list can be passed to the image generation script when you want to generate themes for just a single package - ordinarily the script generates images for all themes, but passing this list as the value of FUSSY-EXCLUDED-THEMES will ensure that they are not loaded during image gen.") (defun generate-elisp-to-fetch-and-exclude (package-names) (let ((downcaser (a:compose #'string-downcase #'symbol-name))) (format nil "(setq fussy-theme-packages-to-load '(~{~a~^ ~}) fussy-excluded-themes '(~{~a~^ ~}))" (mapcar downcaser package-names) (mapcar downcaser +standard-themes+)))) (defun fussy-elisp-script-location (&optional (config *config*)) (uiop:merge-pathnames* (config-directory) (fussy-el config))) (defun format-emacs-evocation-script (package-names) (format nil "env HOME=~a xvfb-run -a emacs -q --eval ~s --load ~a" (full-theme-image-directory) (generate-elisp-to-fetch-and-exclude package-names) (fussy-elisp-script-location))) (defun generate-images-for-packages (package-names) (when package-names (uiop:run-program (format-emacs-evocation-script package-names)))) (defvar *server* nil) (defun image-files-for-theme (theme) (uiop:directory-files (format nil "~a/~a/~a/" (full-theme-image-directory) (string-downcase (theme-pkg-name (theme-package theme))) theme))) (defun theme-preview-image-path (theme &optional (variant "lisp")) (assert (theme-p theme)) (let* ((image-files (image-files-for-theme theme)) (file (or (find variant image-files :key (a:compose #'pathname-type #'pathname-name) :test #'string-equal) (first image-files)))) (format nil "/images/~a/~a/~a.svg" (string-downcase (theme-pkg-name (theme-package theme))) theme (pathname-name file)))) (defun the-update-time () (multiple-value-bind (sec min hour date month year) (get-decoded-time) (declare (ignore min hour)) (destructuring-bind (hh mm) (fetch-time *config*) (encode-universal-time sec mm hh date month year)))) (defun time-to-update-p () (<= (abs (- (get-universal-time) (the-update-time))) 60)) (defvar *fetcher-thread* nil) (defun start-fetch-thread () (setf *fetcher-thread* (bt:make-thread (lambda () (fussy-log "Starting Fetcher Thread") (loop :while t :when (time-to-update-p) :do (handler-case (update-theme-packages) (error (e) (fussy-log "While updating: ~a" e))) :do (sleep 60))) :name "Fussy fetch thread"))) (defun start (&key config-file) (unless config-file (setf config-file (or (uiop:getenv "FUSSY_CONFIG") (asdf:system-relative-pathname :fussy "config.sexp")))) (unless *config* (setf *config* (load-config-from-file config-file) *config-directory* (uiop:pathname-directory-pathname config-file))) (unless *server* (setf *server* (lzb:create-server :port (port *config*) :address (address *config*) :domain (domain *config*)))) (create-db) ;(update-theme-packages) (reindex-themes-by-package) ;(start-fetch-thread) (lzb:install-app *server* (lzb:app 'fussy::fussy)) (lzb:start-server *server*)) (lzb:provision-app () :title "Emacs Themes Library" :content-type "text/html") ;;; pages (defun an-integer (string) (parse-integer string)) (defparameter +themes-per-page+ 20) (defun nav () (html:with-html (:nav (:a :href "/" (:h1 "FUSSY")) (:a :href "/themes" (:h4 "Browse by theme")) (:a :href "/" (:h4 "Browse by theme package")) (random-theme-link)))) (defun page-nav (page page-count &optional terms) "relative pagination nav to current page." (flet ((format-string (pg) (if terms (format nil "?page=~a&terms=~{~a~^+~}" pg terms) (format nil "?page=~a" pg)))) (html:with-html (:div :class "bigger center" (:span (when (plusp page) (:a :href (format-string (1- page)) " <<--- ")) " | " (when (< page (1- page-count)) (:a :href (format-string (1+ page)) " --->> ")))) (:div :class "page-nav" (dotimes (pg page-count) (if (= pg page) (:span (format nil " ~a " (1+ pg))) (:span " " (:a :href (format-string pg) (format nil "~a" (1+ pg))) " "))))))) (defun theme-preview-image (theme &optional (variant "lisp")) (html:with-html (:img :src (theme-preview-image-path theme variant)))) (defun theme-preview-card (theme) (html:with-html (:div :class "card" (:a :href (url-path theme) (:h4 theme) (theme-preview-image theme "lisp"))))) (defun package-preview-card (pkg) (let ((themes (themes-in-package pkg))) (html:with-html (:div :class "card" (:a :href (url-path pkg) (:h4 (format nil "~a - ~D variant~:P" (package-namestring pkg) (length themes))) (theme-preview-image (first (themes-in-package pkg)))))))) (defun style () (html:with-html (:link :rel "stylesheet" :type "text/css" :href "/css/style.css"))) (defparameter +style-css+ " /* CSS Reset */ html { box-sizing: border-box; font-size: 16px; } *, *:before, *:after { box-sizing: inherit; } body, h1, h2, h3, h4, h5, h6, p, ol, ul { margin: 0; padding: 0; font-weight: normal; } ol, ul { list-style: none; } img { max-width: 100%; height: auto; } /* Custom CSS */ html, body { height: 100%; } .main { height: 100%; width: 100%; background-color: #fafffa; } .container { width: 100%; background-color: #fafffa; display: flex; flex-wrap: wrap; justify-content: space-evenly; } .card { min-width: 500px; max-width: 50%; } .card > a > h4 { color: green; text-align: center; } .bigger { font-size: 1.25em; } .center { text-align: center; } .search { margin-left: auto; margin-right: auto; width: 300px; } h1, h2, h3, h4 { text-align: center; } .header { padding-top: 20px; padding-bottom: 20px; margin-top: 20px; margin-bottom: 40px; } .page-nav { text-align: center; font-size: 1.3em; margin-top: 10px; margin-bottom: 20px; } ") (lzb:defendpoint* :get "/css/style.css" () () "Endpoint generating and serving main css" (setf (lzb:response-header :content-type) "text/css") +style-css+) (lzb:defendpoint* :get "/robots.txt" () () "Endpoint for serving robots.txt" (setf (lzb:response-header :content-type) "text/plain") (or (uiop:file-exists-p (robots-path-name)) "")) (defun theme-image-pathname (pkg theme img) (uiop:ensure-pathname (format nil "~a/~a/~a/~a" (full-theme-image-directory) pkg theme img))) (lzb:defendpoint* :get "/images/:pkg:/:theme:/:image:" () () "Endpoint for serving theme preview images" (a:if-let (file-path (theme-image-pathname pkg theme image)) file-path (lzb:http-err 404 "No such file"))) (defun a-string (s) s) (defun a-search-query (terms) "Parses a space separated list for search terms and returns a list." (when terms (str:split-omit-nulls #\space terms))) (defun page-count (things) (ceiling (/ (cond ((numberp things) things) ((listp things) (length things)) (t (error "cant count that"))) +themes-per-page+))) (defun get-page (page things) (a:subseq* things (* page +themes-per-page+) (* (1+ page) +themes-per-page+))) (lzb:defendpoint* :get "/themes" ((page an-integer) (terms a-search-query)) () "The landing page" (let* ((page (or page 0)) (all-themes (if terms (search-themes terms :sort-by (complement #'package-version<=)) (all-themes :sort-by (complement #'package-version<=)))) (page-count (page-count all-themes)) (themes (get-page page all-themes))) (html:with-html-string (:doctype) (:html (:head (:title "Fussy - Emacs Themes Gallery") (style)) (:body (:div :class "main" (nav) (gallery-header (length all-themes) "themes") (search-form "/themes" terms) (when (< 1 page-count) (page-nav page page-count terms)) (:div :class "container" (dolist (theme themes) (theme-preview-card theme))) (:div :class "foooter" (when (< 1 page-count) (page-nav page page-count terms))))))))) (defun search-form (endpoint terms) (html:with-html (:div :class "search" (:form :method "GET" :action endpoint (:label :for "terms") (:input :name "terms" :value (format nil "~{~a ~}" terms)) (:button :type "submit" "search"))) )) (defun gallery-header (count what) (html:with-html (:div :class "center header" (:p "Emacs Themes Gallery for your Daily Procrastination Needs." (:br) "Showing " (write-to-string count) " " what " from " (:a :href "https://melpa.org" "MELPA.") (:br) "Newest first.")))) (defun random-theme-link () (html:with-html (:div :class "center" (:a :href "/random-theme" "Random Theme")))) (defun redirect-to (location) "Set the lazybones response header and response code for redirecting to LOCATION. This procedure will error if lazybones:*request* is not currently bound." (setf (lzb:response-header :location) location (lzb:response-code) 303) "") (defun select-random (xs) (elt xs (random (length xs)))) (defun random-theme-url () (url-path (select-random (all-themes)))) (lzb:defendpoint* :get "/random-theme" () () "Return a random theme url" (redirect-to (random-theme-url))) (lzb:defendpoint* :get "/" ((page an-integer) (terms a-search-query)) () "Browse the packages" (let* ((page (or page 0)) (all-packages (if terms (search-packages terms :sort-by (complement #'package-version<=)) ;; temporary hack b/c of a bug with non-updated themes (search-packages '("theme") :sort-by (complement #'package-version<=)) ;;(all-theme-packages (complement #'package-version<=)) )) (page-count (ceiling (/ (length all-packages) +themes-per-page+))) (packages (get-page page all-packages))) (html:with-html-string (:doctype) (:html (:head (:title "Fussy - Packages") (style)) (:body (:div :class "main" (nav) (gallery-header (length all-packages) "theme packages") (search-form "/" terms) (when (< 1 page-count) (page-nav page page-count terms)) (:div :class "container" (dolist (pkg packages) (package-preview-card pkg))) (:div :class "footer" (when (< 1 page-count) (page-nav page page-count terms))))))))) (defun a-package (string) (a:if-let (pkg (theme-pkg-with-name (intern (string-upcase string) :fussy))) pkg (lzb:http-err 404 (format nil "No such theme package: ~a" string)))) (defun a-theme (string) (if (theme-p string) string (lzb:http-err 404 (format nil "No such theme: ~a" string)))) (defun package-namestring (pkg) (hq:>> () pkg theme-pkg-name symbol-name string-downcase)) (lzb:defendpoint* :get "/package/:pkg a-package:" ((page an-integer)) () "The page endpoint for a theme package" (let ((themes (themes-in-package pkg)) (page (or page 0))) (html:with-html-string (:doctype) (:html (:head (:title ) (style)) (:body (:div :class "main" (nav) (:div :class "center header" (:h2 (package-namestring pkg) ) (:p (theme-pkg-description pkg)) (:p "This package contains" (length themes) (if (= 1 (length themes)) " theme" " themes")) (:p (:a :href (theme-pkg-url pkg) (theme-pkg-url pkg)))) (when (< +themes-per-page+ (length themes)) (page-nav page (floor (/ (length themes) +themes-per-page+)))) (:div :class "container" (dolist (theme (a:subseq* themes (* page +themes-per-page+) (* (1+ page) +themes-per-page+))) (theme-preview-card theme))) (when (< +themes-per-page+ (length themes)) (page-nav page (floor (/ (length themes) +themes-per-page+)))))))))) (lzb:defendpoint* :get "/theme/:pkg a-package:/:theme a-theme:" () () "The page for a particular theme showing its previews for different prog langs" (unless (member theme (themes-in-package pkg) :test #'string-equal) (lzb:http-err 403 (format nil "The theme ~a is not in the package ~a" theme (package-namestring pkg)))) (html:with-html-string (:doctype) (:html (:head (:title theme) (style)) (:body (:div :class "main" (nav) (:div :class "center header" (:h2 theme) (:p "A theme in the package " (:a :href (url-path pkg) (package-namestring pkg)))) (:div :class "container" (dolist (variant '("lisp" "py" "cpp")) (theme-preview-image theme variant))))))))