From: Ian Kelling <ian@iankelling.org>
Date: Wed, 15 Feb 2023 02:13:12 +0000 (-0500)
Subject: speed up nav2beet, 8 mins to 1 min
X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=commitdiff_plain;h=3c7dab4936f011bb91f42f7ccb1f8432b94ea040

speed up nav2beet, 8 mins to 1 min
---

diff --git a/nav2beet b/nav2beet
index b0ffcd5..cc9ca15 100755
--- a/nav2beet
+++ b/nav2beet
@@ -2,16 +2,36 @@
 
 f=/usr/local/lib/err;test -r $f || { echo "error: $0 no $f" >&2;exit 1;}; . $f
 
+plists=(
+  # these are useful tags
+  expl
+  gimicky
+  sad
+  # these are normal playlists
+  love
+  pump1
+  pumprap
+  rend
+  run
+)
+
 
+# these options are mainly for debugging / developing quickly.
 plist_only=false
 dry_run=false
 while (( $# )); do
   case $1 in
     plist)
       plist_only=true
+      shift
       ;;
     dry)
       dry_run=true
+      shift
+      ;;
+    *)
+      echo "error. unexpected arg read script"
+      exit 1
       ;;
   esac
 done
@@ -28,37 +48,45 @@ m ()
 
 tmpf=$(mktemp)
 
+declare -A flacs
 
-if ! $plist_only; then
-  echo begin star rating import from navidrome to beets
-
-  declare -A navirating
-  tmpdir=$(mktemp -d)
-  cd $tmpdir
-  if [[ $HOSTNAME != kd ]]; then
-    ssh_prefix="ssh b8.nz"
-    ssh b8.nz '
+declare -A navirating
+tmpdir=$(mktemp -d)
+cd $tmpdir
+if [[ $HOSTNAME != kd ]]; then
+  ssh b8.nz bash -s <<EOF  | tar xz
 install -m 700 -d /tmp/nav2beet
+cd /tmp/nav2beet
 for r in 1 2 3 4 5; do
- sqlite3 /i/navidrome/navidrome.db ".output /tmp/nav2beet/$r" "select path from annotation inner join media_file on item_id = id where rating = $r;"
+ sqlite3 /i/navidrome/navidrome.db ".output \$r" "select path from annotation inner join media_file on item_id = id where rating = \$r;"
+done
+find /i/m -type f -name '*.flac' >flacs
+for plist in ${plists[@]}; do
+  sqlite3 /i/navidrome/navidrome.db "select path from media_file inner join playlist_tracks on media_file.id = media_file_id where playlist_id = (select id from playlist where name = '\$plist');" | sed 's,^/i/converted,/i/m,' | sort >\$plist
 done
 tar cz -C /tmp nav2beet
-' | tar xz
-    cd nav2beet
-  else
-    for r in 1 2 3 4 5; do
-      sqlite3 /i/navidrome/navidrome.db ".output $r" "select path from annotation inner join media_file on item_id = id where rating = $r;"
-    done
-  fi
-  declare -A flacs
+EOF
+
+  cd nav2beet
+else
+  for r in 1 2 3 4 5; do
+    sqlite3 /i/navidrome/navidrome.db ".output $r" "select path from annotation inner join media_file on item_id = id where rating = $r;"
+  done
+  find /i/m -type f -name '*.flac' >flacs
+  for plist in ${plists[@]}; do
+    sqlite3 /i/navidrome/navidrome.db "select path from media_file inner join playlist_tracks on media_file.id = media_file_id where playlist_id = (select id from playlist where name = '$plist');" | sed 's,^/i/converted,/i/m,' | sort >$plist
+  done
+fi
+while read -r l; do
+  flacs[$l]=t
+done <flacs
 
+if ! $plist_only; then
+  echo begin star rating import from navidrome to beets
   # todo: consider if this is a problem: file removed/renamed in main
   # collection, but not yet updated navidrome, we want to skip it not
   # die.
 
-  while read -r l; do
-    flacs[$l]=t
-  done < <($ssh_prefix find /i/m -type f -name '*.flac')
   for r in 1 2 3 4 5; do
     while read -r path; do
       beetpath="/i/m${path#/i/converted}"
@@ -69,8 +97,6 @@ tar cz -C /tmp nav2beet
       navirating[$beetpath]=$r
     done <$r
   done
-  cd
-  rm -rf $tmpdir
   declare -A beetrating
   for r in 1 2 3 4 5; do
     m beet ls -f '$path' rating:$r >$tmpf
@@ -90,75 +116,36 @@ tar cz -C /tmp nav2beet
   # end star rating import from navidrome to beets:
 fi
 
+
 echo begin import navidrome playlists as flexible attribute with value t.
 # These are only the playlists listed in the beets config.yaml
 # "subsonicplaylist:" and then duplicated here:
 
 
-plists=(
-  # these are useful tags
-  expl
-  gimicky
-  sad
-  # these are normal playlists
-  love
-  pump1
-  pumprap
-  rend
-  run
-)
-# this puts the navidrome playlists into a smart attribute
-# subsonic_playlist
-m beet subsonicplaylist
 
-beet ls -f '$id $subsonic_playlist' subsonic_playlist::. | sed 's/;/ /g' >$tmpf
-
-# for debugging
-#m head $tmpf
-
-navlists=()
-while read -r id id_plists; do
-  navlists[id]="$id_plists"
-done <$tmpf
 
 for plist in ${plists[@]}; do
   echo "processing $plist"
-  for id in $(beet ls -f '$id' $plist:t ^genre:spoken-w ^genre:skit ^rating:1); do
-    found=false
-    newnavlist=()
-    for navlist in ${navlists[id]}; do
-      if [[ $navlist == "$plist" ]]; then
-        found=true
-      else
-        newnavlist+=($navlist)
-      fi
-    done
-    if $found; then
-      navlists[id]="${newnavlist[*]}"
-    else
-      # exists in beets, but not navidrome so we must have removed
-      # it from the playlist in navidrome.
-      m beet modify -y "id:$id" "$plist!"
+  beet ls -f '$path' $plist:t ^genre:spoken-w ^genre:skit ^rating:1 | sort | sed 's,\.flac$,.mp3,'> p
+  while read -r path; do
+    flac="${path%.mp3}.flac"
+    if [[ ${flacs[$flac]} ]]; then
+      path="$flac"
     fi
-  done
-done
-# The ones we didnt find and remove are ones we added
-# in navidrome.
-for id in ${!navlists[@]}; do
-  [[ ${navlists[id]} ]] || continue
-  m beet modify -y "id:$id" ${navlists[id]// /=t }=t
-done
-
-# old way of doing it which sets everything. this doesnt
-# account for removed tracks in navidrome, and it will
-# be slower for handling many playlists.
-# while read -r id plists; do
-#   plists="${plists#;}"
-#   m beet modify -y "id:$id" ${plists//;/=t }
-# done < <(beet ls -f '$id $subsonic_playlist' subsonic_playlist::.)
+    m beet modify -y "path:$path" "$plist!"
+    # files unique to tmpf are in beets not navidrome
+  done < <(comm -23 p $plist)
+  while read -r path; do
+    flac="${path%.mp3}.flac"
+    echo "flac=$flac path=$path ${flacs[$flac]} ${flacs[flac]}"
+    if [[ ${flacs[$flac]} ]]; then
+      path="$flac"
+    fi
+    m beet modify -y "path:$path" $plist=t
+    # files unique to plist are in navidrome not beets
+  done < <(comm -13 p $plist)
 
-# if we remove a track from a playlist in navidrome,
-# beet subsonicplaylist won't remove corresponding beet
-# smart attribute, so clear them all here.
+done
 
-m beet modify -y subsonic_playlist::. 'subsonic_playlist!'
+cd
+rm -rf $tmpdir