X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mw-setup-script;h=599766800be8cc5fd03f44723c68fec04092a8ee;hb=36be2f22108098545ff35f189db1637102febcdb;hp=151d9495643ede6662fa481adbf91a318abb101f;hpb=90876e4723500bf439f2d6a0009c8c9b654cbff2;p=mediawiki-setup

diff --git a/mw-setup-script b/mw-setup-script
index 151d949..5997668 100755
--- a/mw-setup-script
+++ b/mw-setup-script
@@ -1,81 +1,119 @@
 #!/bin/bash
 # Copyright (C) 2016 Ian Kelling
 # This program is under GPL v. 3 or later, see <http://www.gnu.org/licenses/>
-# <source lang="bash">
-# for convenience, Mediawiki config file
-mwc="$mw/LocalSettings.php"
-
-if isdeb; then
-    apache_user=www-data
-else
-    apache_user=apache
-fi
+set -x
+# <source lang="bash">
+# identify if this is a debian based distro
+isdeb() { command -v apt &>/dev/null; }
+# tee unique. append each stdin line if it does not exist in the file
+teeu () {
+    local MAPFILE
+    mapfile -t
+    for line in "${MAPFILE[@]}"; do
+        grep -xFq "$line" "$1" &>/dev/null || tee -a "$1" <<<"$line"
+    done
+}
 
 # get and reset an extension/skin repository, and enable it
-mw-ext () { mw-extra extension $@; }
-mw-skin() { mw-extra skin $@; }
-
-mw-extra() {
-    local type=${1}s # extension or skin
-    shift
-    local clone=true
-    if [[ $1 == -g ]]; then
-        clone=false
-        shift
+mw-clone() {
+    local url=$1
+    local original_pwd="$PWD"
+    local name
+    local re='[^/]*/[^/]*$'
+    [[ $url =~ $re ]] ||:
+    target=$mw/${BASH_REMATCH[0]}
+    if [[ ! -e $target/.git ]]; then
+        git clone $url $target
     fi
+    if ! cd $target; then
+        echo "mw-ext error: failed cd $target";
+        exit 1
+    fi
+    git fetch
+    git checkout -qf origin/$mw_branch || git checkout -qf origin/master
+    git clean -xffd
+    cd "$original_pwd"
+
+}
+mw-ext () {
     local ext
     for ext in "$@"; do
-        if $clone; then
-            local original_pwd="$PWD"
-            # it's ok that this fails if we already have it
-            url=https://git.wikimedia.org/git/mediawiki
-            target=$mw/$type/$ext
-            git clone $url/$type/$ext.git $target
-            if ! cd $target; then
-                echo "mw-ext error: failed cd $mw/extensions/$ext";
-                exit 1
-            fi
-            git fetch
-            git checkout -qf origin/$mw_branch || git checkout -qf origin/master
-            git clean -xffd
-            cd "$original_pwd"
-        fi
-        case $type in
-            extensions)
-                if [[ -e $target/extension.json ]]; then
-                    # new style extension. remove old style declaration
-                    sed -i '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc
-                    teeu $mwc <<EOF
+        mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/$ext
+        if [[ -e $mw/ext/$ext/extension.json ]]; then
+            # new style extension. remove old style declaration
+            sed -i --follow-symlinks '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc
+            teeu $mwc <<EOF
 wfLoadExtension( '$ext' );
 EOF
-                else
-                    teeu $mwc <<EOF
+        else
+            teeu $mwc <<EOF
 require_once( "\$IP/extensions/$ext/$ext.php" );
 EOF
-                fi
-                ;;
-            skins)
-                sed -i '/^wfLoadSkin/d' $mwc
-                sed -i '/^\$wgDefaultSkin/d' $mwc
-                teeu $mwc <<EOF
-$wgDefaultSkin = "${ext,,*}";
-wfLoadSkin( 'Vector' );
-EOF
-                ;;
-        esac
+        fi
     done
     # --quick is quicker than default flags,
     # but still add a sleep to make sure everything works right
     sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
 }
+mw-skin() {
+    local skin=$1
+    mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/skins/$skin
+    sed -i --follow-symlinks '/^wfLoadSkin/d' $mwc
+    sed -i --follow-symlinks '/^\$wgDefaultSkin/d' $mwc
+    teeu $mwc <<EOF
+\$wgDefaultSkin = "${skin,,*}";
+wfLoadSkin( '$skin' );
+EOF
+    sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
+}
+
+if command -v apt &>/dev/null; then
+    apache_user=www-data
+else
+    apache_user=apache
+fi
+
 # </source>
 # <source lang="bash">
+# From here on out, exit if a command fails.
+# This will prevent us from not noticing an important failure.
+# We recommend setting this for the entire installation session.
+# If you are running commands interactively, it might be best to
+# put it in your ~/.bashrc temporarily.
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
+source ~/mw_vars
+
 if isdeb; then
+    # main reference:
+    # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu
     apt-get update
-    # noninteractive to avoid mysql password prompt
-    DEBIAN_FRONTEND=noninteractive apt-get -y install mediawiki ImageMagick php5-mysqlnd php-apc
+    DEBIAN_FRONTEND=noninteractive apt-get install -y imagemagick curl
+    if apt-get install -s mediawiki &>/dev/null; then
+        # mediawiki is packaged in jessie backports.
+        DEBIAN_FRONTEND=noninteractive apt-get -y install php5-apcu mediawiki
+    else
+        # https://www.mediawiki.org/wiki/Manual:Installation_requirements
+        if apt-get install -s php7.0 &>/dev/null; then
+            # note, 7.0 is untested by the editor here, since it's not
+            # available in debian 8. it's listed as supported
+            # in the mediawiki page.
+            # noninteractive to avoid mysql password prompt.
+            DEBIAN_FRONTEND=noninteractive apt-get install -y apache2  \
+                           default-mysql-server \
+                           php7.0 php7.0-mysql libapache2-mod-php7.0 php7.0-xml \
+                           php7.0-apcu php7.0-mbstring
+        else
+            # note: mbstring is recommended, but it's not available for php5 in
+            # debian jessie.
+            DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 \
+                           default-mysql-server \
+                           php5 php5-mysql libapache2-mod-php5 php5-apcu
+        fi
+    fi
     service apache2 restart
 else
+    # note
     # fedora deps are missing a database, so some is translated from debian packages
     yum -y install mediawiki ImageMagick php-mysqlnd php-pecl-apcu mariadb-server
 
@@ -103,15 +141,21 @@ fi
 mkdir -p $mw
 cd $mw
 # this will just fail if it already exists which is fine
-git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
+if [[ ! -e .git ]]; then
+    git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
+fi
 # to see available branches: https://www.mediawiki.org/wiki/Version_lifecycle
 # and
 # git branch -r
 git checkout -f origin/$mw_branch
 git clean -ffxd
+# add some librejs support:
+curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-librejs-patch;hb=HEAD" | bash
 # Get the php libraries wmf uses. Based on:
 # https://www.mediawiki.org/wiki/Download_from_Git#Fetch_external_libraries
-git clone https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
+if [[ ! -e vendor/.git ]]; then
+    git clone  https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
+fi
 cd vendor
 git checkout -f origin/$mw_branch
 cd ..
@@ -119,7 +163,7 @@ cd ..
 # Drop any previous database which may have been installed while testing.
 # If upgrading, we should have a db backup which will get restored.
 # https://www.mediawiki.org/wiki/Manual:Upgrading
-mysql -u root -p$dbpass <<'EOF'
+mysql -u root -p$dbpass <<'EOF' ||:
 drop database my_wiki;
 exit
 EOF
@@ -134,56 +178,31 @@ $wgGroupPermissions['*']['createaccount'] = false;
 EOF
 # </source>
 # <source lang="bash">
-tmpdir="$(mktemp -d)"
-cd $tmpdir
-wget http://builds.piwik.org/piwik.zip
-if isdeb; then apt-get -y install unzip; else yum -y install unzip; fi
-unzip -q piwik.zip
-# gui installer suggested command
-if isdeb; then
-    chown -R www-data:www-data piwik
-else
-    chown -R apache:apache piwik
-fi
-# remove any existing directory
-rm -rf $mw/../analytics
-mv piwik $mw/../analytics
-rm -rf $tmpdir
-
-# </source>
-# <source lang="bash">
+temp=$(mktemp -d)
+cd $temp
+git_site=https://iankelling.org/git
+git clone $git_site/acme-tiny-wrapper
 l=$mw/../../logs
-mkdir $l
+mkdir -p $l
 
-acme-tiny-wrapper $mwdomain
-dd of=/etc/apache2/ports.conf <<'EOF'
-Listen 8082
-EOF
+acme-tiny-wrapper/acme-tiny-wrapper -t $mwdomain
+
+git clone $git_site/basic-https-conf
 { cat <<EOF
-  ServerAdmin $mw_email
-  RewriteEngine On
-  # make the site's root url go to our main page
-  RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
-  # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
-  RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
-EOF
-  find -L $(readlink -f $mw/..) -name .htaccess \
-      | while read line; do
-      echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
-  done
-} | apache-site -i -p 127.0.0.1:8082 -- - $mwdomain
-nginx-site -p 8082 $mwdomain
-# </source>
-# <source lang="bash">
-if isdeb; then
-    a2ensite $mwdomain.conf
-    # for short urls
-    a2enmod rewrite
-    # We restart rather than reload because of the module
-    service apache2 restart
-else
-    systemctl reload httpd.service
-fi
+ServerAdmin $mw_email
+RewriteEngine On
+# make the site's root url go to our main page
+RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
+# use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
+RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
+EOF
+find -L $(readlink -f $mw) -name .htaccess \
+    | while read line; do
+    echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
+done
+} | basic-https-conf/apache-site -r ${mw%/*} - $mwdomain
+cd
+rm -rf $temp
 # </source>
 # <source lang="bash">
 dd of=$mw/../robots.txt <<'EOF'
@@ -192,6 +211,7 @@ Disallow: /w/
         User-agent: ia_archiver
 Allow: /*&action=raw
 EOF
+mw-skin Vector
 # </source>
 # <source lang="bash">
 teeu $mwc<<EOF
@@ -200,7 +220,6 @@ teeu $mwc<<EOF
 \$wgRightsUrl = "$mw_RightsUrl";
 \$wgRightsText = "$mw_RightsText";
 \$wgRightsIcon = "$mw_RightsIcon";
-wfLoadSkin( 'Vector' );
 EOF
 # </source>
 # <source lang="bash">
@@ -212,6 +231,7 @@ teeu $mwc<<EOF
 \$wgMainCacheType = CACHE_ACCEL;
 \$wgEnableUploads = true;
 \$wgUseInstantCommons = true;
+\$wgPingback = true;
 EOF
 # </source>
 # <source lang="bash">
@@ -224,20 +244,22 @@ $wgArticlePath = "/wiki/$1";
 # not using nofollow is good practice, as long as we avoid spam.
 $wgNoFollowLinks = false;
 # Allow user customization.
-$wgAllowUserJs = true;
 $wgAllowUserCss = true;
-
 # use imagemagick over GD
 $wgUseImageMagick = true;
+# manual says this is not production ready, I think that is mostly
+# because they are using MobileFrontend extension instead, which gives
+# an even cleaner more minimal view, I plan to try setting it up
+# sometime but this seems like a very nice improvement for now.
+$wgVectorResponsive = true;
 EOF
 
 
 # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads
 # Increase from default of 2M to 100M.
 # This will at least allow high res pics etc.
-php_ini=$(isdeb && echo /etc/php5/apache2/php.ini || echo /etc/php.ini)
-sed -i 's/^\(upload_max_filesize\)\b.*/\1 = 100M/'
-sed -i 's/^\(post_max_size\)\b.*/\1 = 100M/'
+php_ini=$(php -r 'echo(php_ini_loaded_file());')
+sed -i --follow-symlinks 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini
 if isdeb; then
     service apache2 restart
 else
@@ -254,18 +276,19 @@ chmod -R g+w $mw/images
 # <source lang="bash">
 teeu $mwc <<'EOF'
 $wgLogo = null;
-$wgShowIPinHeader = false;
-$wgFooterIcons = null;
+#$wgFooterIcons = null;
 EOF
 # Make the toolbox go into the drop down.
-cd $mw
-git remote add ian git@gitorious.org:mediawiki-toolbox-patch/mediawiki-toolbox-patch.git
-git fetch ian
-git rebase ian/REL1_23-toolbox-in-dropdown
+cd $mw/skins/Vector
+if ! git remote show ian-kelling &>/dev/null; then
+    git remote add ian-kelling https://iankelling.org/git/forks/Vector
+fi
+git fetch ian-kelling
+git checkout ian-kelling/${mw_branch}-toolbox-in-dropdown
 # </source>
 # <source lang="bash">
-mw-ext Cite CiteThisPage CSS DynamicPageList Echo Gadgets ImageMap Interwiki News \
-       Nuke ParserFunctions Poem SyntaxHighlight_GeSHi Variables
+mw-ext Cite CiteThisPage CheckUser CSS Echo Gadgets ImageMap Interwiki News \
+       Nuke ParserFunctions Poem Renameuser SyntaxHighlight_GeSHi Variables
 # </source>
 # <source lang="bash">
 mw-ext AntiSpoof
@@ -273,16 +296,13 @@ mw-ext AntiSpoof
 sudo -u $apache_user php $mw/extensions/AntiSpoof/maintenance/batchAntiSpoof.php
 # </source>
 # <source lang="bash">
-mw-ext CheckUser
-sudo -u $apache_user php $mw/extensions/CheckUser/install.php; sleep 1
-# </source>
-# <source lang="bash">
 if isdeb; then
     apt-get -y install php-wikidiff2
     teeu $mwc <<'EOF'
 $wgExternalDiffEngine = 'wikidiff2';
 EOF
-    ln -sf ../../mods-available/wikidiff2.ini /etc/php5/apache2/conf.d
+    dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d
+    ln -sf ../../mods-available/wikidiff2.ini $dir
     service apache2 restart
 fi
 # </source>
@@ -290,12 +310,16 @@ fi
 mw-ext Math
 # php5-curl according to Math readme
 if isdeb; then
-    apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base ghostscript imagemagick ocaml php5-curl
+    curl_pkg=php7.0-curl
+    if ! apt-get -s install $curl_pkg &>/dev/null; then
+        curl_pkg=php5-curl
+    fi
+    apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base \
+            ghostscript imagemagick ocaml $curl_pkg make
 else
     # todo, php5-curl equivalent on fedora
     yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml
 fi
-ln -sf ../../mods-available/curl.ini /etc/php5/apache2/conf.d
 service apache2 restart
 
 cd $mw/extensions/Math/math; make # makes texvc
@@ -391,19 +415,21 @@ EOF
 fi
 # </source>
 # <source lang="bash">
-sed -i "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
+sed -i --follow-symlinks "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
 # </source>
 # <source lang="bash">
 # get repo
-mkdir ~/opt
-git clone --recursive https://gerrit.wikimedia.org/r/pywikibot/core.git ~/opt/pywikibot
-cd ~/opt/pywikibot
+if [[ ! -e ~/pywikibot/.git ]]; then
+    git clone --recursive \
+        https://gerrit.wikimedia.org/r/pywikibot/core.git ~/pywikibot
+fi
+cd ~/pywikibot
 #updating
 git pull --all
 git submodule update
 # </source>
 # <source lang="bash">
-cd $HOME/opt/pywikibot
+cd $HOME/pywikibot
 dd of=user-config.py <<EOF
 mylang = 'en'
 usernames["$mwfamily"]['en'] = u'$wikiuser'
@@ -418,6 +444,12 @@ EOF
 
 # it won't overrwrite an existing file. Remove if if one exists
 rm -f  pywikibot/families/${mwfamily}_family.py
+if isdeb; then
+    apt-get install -y python-requests
+else
+    yum -y install python-requests
+fi
+
 python generate_family_file.py https://$mwdomain/wiki/Main_Page "$mwfamily"
 
 # Note, this needed only for ssl site
@@ -427,9 +459,9 @@ tee -a pywikibot/families/${mwfamily}_family.py<<'EOF'
 EOF
 # </source>
 # <source lang="bash">
-pw="$HOME/opt/pywikibot"
+cd "$HOME/pywikibot"
 
-dd of=$pw/scripts/${mwfamily}_setup.py<<EOF
+dd of=scripts/${mwfamily}_setup.py<<EOF
 import pywikibot
 import time
 import sys
@@ -505,116 +537,40 @@ div#mw-content-text {
 """)
 EOF
 
-cd $pw
-python pwb.py ${mwfamily}_setup
-# </source>
-# <source lang="bash">
-#!/bin/bash
-source ~/mw_vars
-
-# if we get an error, keep going but return it at the end
-last_error=0
-trap 'last_error=$?' ERR
-
-ssh root@$mwdomain <<ENDSSH
-tee -a $mwc<<'EOF'
-$wgReadOnly = 'Dumping Database, Access will be restored shortly';
-EOF
-mysqldump -p$dbpass --default-character-set=binary my_wiki  > ~/wiki_backup/wikidump
-sed -i '$ d' $mwc # delete last line
-ENDSSH
-rdiff-backup root@$mwdomain::/root/wiki_db_backup ~/backup/wiki_db_backup
-rdiff-backup root@$mwdomain::$mw ~/backup/wiki_file_backup
-
-exit $last_error
+# this can spam a warning, so uniq it
+python pwb.py ${mwfamily}_setup |& uniq
 # </source>
 # <source lang="bash">
-chmod +x $HOME/bin/remote_wiki_backup
-x="$(mktemp)"
-crontab -l > "$x"
-teeu "$x" <<'EOF'
-0 4 * * * x=$($HOME/bin/remote_wiki_backup 2>&1); [[ $? != 0 ]] && echo "$x"
-EOF
-crontab "$x"
-# </source>
-# <source lang="bash">
-source ~/mw_vars
-HOSTNAME=REPLACE_ME source ~/mw_vars
-rdiff-backup -r now ~/backup/wiki_file_backup /tmp/wiki_file_restore
-scp -r /tmp/wiki_file_restore/images root@$mwdomain:$mw/images
-rdiff-backup -r now ~/backup/wiki_db_backup /tmp/wiki_db_restore
-scp -r /tmp/wiki_db_restore root@$mwdomain:/tmp
-ssh root@$mwdomain "mysql -u root -p$dbpass my_wiki < /tmp/wiki_db_restore/wiki_db_dump"
-# </source>
-# <source lang="bash">
-s=$HOME/bin/mediawiki_update
+s=/etc/cron.daily/mediawiki_update
 dd of=$s<<'EOF'
 #!/bin/bash
 source ~/mw_vars
-cd $mw
-git fetch --all
-git checkout origin/$mw_branch
-git rebase ian/REL1_23-toolbox-in-dropdown
-cd extensions
-for x in *; do
-    if [[ -d $x ]]; then
-        cd $x
-        git fetch --all
-        git checkout origin/$mw_branch || git checkout -qf origin/master
-        cd ..
+update() {
+    dir=$1
+    cd $mw
+    [[ -d $dir ]] || return 1
+    cd $dir
+    branch=$(git describe --all)
+    branch=${branch#remotes/}
+    git fetch --all -q
+    new_head=$(git rev-parse $branch)
+    log=$(git log HEAD..$new_head)
+    if [[ ! $log ]]; then
+        return 1
     fi
+    pwd
+    echo "$log"
+    git checkout -qf $new_head
+    cd $mw
+    return 0
+}
+for dir in extensions/* skins/* vendor; do
+    update "$dir" ||:
 done
-php $mw/maintenance/update.php -q
-EOF
-chmod +x $s
-
-x="$(mktemp)"
-crontab -l > "$x"
-teeu "$x" <<EOF
-0 5 * * * $s
-EOF
-crontab "$x"
-# </source>
-# <source lang="bash">
-# docs suggests using separate database user, google lead me here for how
-# https://www.digitalocean.com/community/tutorials/how-to-create-a-new-user-and-grant-permissions-in-mysql
-mysql -u root -p$dbpass <<EOF
-CREATE USER 'piwik'@'localhost' IDENTIFIED BY '$piwik_pass';
-GRANT ALL PRIVILEGES ON piwik . * TO 'piwik'@'localhost';
-FLUSH PRIVILEGES;
-exit
-EOF
-# </source>
-# <source lang="bash">
-git clone https://github.com/DaSchTour/piwik-mediawiki-extension.git $mw/extensions/Piwik
-mw-ext -g Piwik
-teeu $mwc <<EOF
-\$wgPiwikURL = '$mwdomain/analytics/';
-\$wgPiwikIDSite = '1';
-EOF
-# </source>
-# <source lang="bash">
-s=~/bin/piwik-update
-mkdir -p ${s%/*}
-dd of=$s <<EOF
-#!/bin/bash
-piwik_path=$mw/../analytics
-cd "\$(mktemp -d)"
-wget -q http://builds.piwik.org/piwik.zip
-unzip -q piwik.zip
-rm -f piwik.zip
-cp \$piwik_path/config/config.ini.php piwik/config
-cp -rf piwik/* \$piwik_path
-# prevent making an email out of the standard success response
-x="\$( \$piwik_path/console core:update )"
-if [[ \$x != "Everything is already up to date." ]]; then
-echo "\$x"
+if update .; then
+    curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-librejs-patch;hb=HEAD" | bash
 fi
+php $mw/maintenance/update.php -q --quick
 EOF
-chmod +x $s
 
-x="$(mktemp)"
-crontab -l > "$x"
-teeu "$x" <<< "0 3 * * tue $s"
-crontab "$x"
 # </source>