From 1a26b348cefa35e6a0a4d0b28624f67609daf49c Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Thu, 23 Jun 2016 00:39:07 -0700 Subject: [PATCH] update to latest snapshot --- Mediawiki_Setup_Guide | 300 +++++++++++++++--------------------------- mw-setup-script | 141 +++++++------------- 2 files changed, 151 insertions(+), 290 deletions(-) diff --git a/Mediawiki_Setup_Guide b/Mediawiki_Setup_Guide index 440beaf..f1353ec 100644 --- a/Mediawiki_Setup_Guide +++ b/Mediawiki_Setup_Guide @@ -1,6 +1,6 @@ == Introduction == -'''tldr''': Automated setup on gnu/linux after you set options in [[#Dependent Variables and Functions]], except email. Any code which is not ready to be run has bold text saying so immediately before it. +'''tldr''': For Gnu/Linux (with a bit of Debian bias), a more concise, holistic and automated install than the official Mediawiki docs. Do some initial configuration then download this page and run it, or execute it as you read the well written documentation. ''' Goals / Why use this guide? ''' @@ -8,7 +8,7 @@ * Closely references & supplements official documentation * Explicit automation support wherever practical * Used to setup this site (style optional) -* Contributions welcome. This site will be updated/tested with changes to this document! +* Contributions welcome and will be updated/tested on this very site! * Support for multiple linux distros * Holistic scope (backups, server setup), but sections stand on their own * Explicit support for production & local testing instance. Additions for production like https and web analytics. @@ -25,10 +25,11 @@ Very minor adjustments needed for other distros. Help expand this list. * Mediawiki 1.27 * Debian 8 (tested) +* Debian testing (tested Aug 7, 2016) Pre 5/2016 revisions ran Mediawiki 1.23, tested on Fedora 20 and Ubuntu 14.04. -== Production Server Prerequisites == +== Prerequisites == '''Getting a Server & a Domain''' @@ -40,32 +41,7 @@ Setting up email can be an involved process. Mediawiki is perfectly happy to dis If you are not setting up your server to send mail with a program that uses the default sendmail interface, see these pages when you are configuring mediawiki: [[mediawikiwiki:Manual:$wgEnableEmail|Manual:$wgEnableEmail]], [https://www.mediawiki.org/wiki/Configuration_settings#Email_settings Manual:Email_settings], [[mediawikiwiki:Manual:$wgSMTP|Manual:$wgSMTP]] -== Stopping Spam == - -There is a balance between effective anti-spam measures and blocking/annoying contributors. Mediawiki documentation on how to combat spam, is not very good, but it has improved over time: [https://www.mediawiki.org/wiki/Manual:Combating_spam manual: Combating Spam]. It's possible for a spammer to quickly make thousands of edits, and there is no good documentation on purging lots of spam, so you should have a good strategy up front. My current strategy is 3 fold, and is limited to small/medium wiki's: - -* Find new spam quickly, revert it & ban the user. -** Watch, and get notified of changes on all primary content pages: Special:Preferences, Bottom of the page, set an email address, then turn on "Email me also for minor edits of pages and files." -** Use a rss/atom feed reader, and subscribe to recent changes across the wiki. Newer browsers have an rss feed subscribe button, you can click after going to Special:RecentChanges. If that is not available, you can construct the proper url based on [https://meta.wikimedia.org/wiki/Help:Recent_changes#Web_feed these instructions]. -* Require registration to edit, and a custom captcha question on registration. -* Install all non-user inhibiting anti-spam extensions / settings that take a reasonable amount of time to figure out. - -== Download all ready to use source code blocks into a single script == - -Source block that are not ready to use have a bold warning just before them (and a tag on the block) and are skipped. -''' Requires customization: outputs to stdout''' - -start=' * *' -end=' *<\/source> *' -ruby <<'EOF' | sed -rn "/^$start$/,/^$end$/{s/^$start|$end$/# \0/;p}" -require 'json' -puts JSON.parse(`curl 'https://ofswiki.org/w/api.php?\ -action=query&titles=Mediawiki_Setup_Guide&prop=revisions&rvprop=content&\ -format=json'`.chomp)['query']['pages'].values[0]['revisions'][0]['*'] -EOF - - -== Dependent Variables and Functions == +== Setup Guide Configuration == # Set variables below # Save the code in this section to a file (~/mw_vars is suggested) @@ -74,26 +50,24 @@ EOF '''Requires customization:''' -# Replease REPLACE_ME as appropriate +# Replace REPLACE_ME as appropriate -export mwdescription="Opinionated Free Software Wiki" +export mwdescription="REPLACE_ME" # eg. Opinionated Free Software Wiki # username/pass of the first wiki admin user -export wikiuser="Ian Kelling" +export wikiuser="REPLACE_ME" export wikipass=REPLACE_ME # root password for the mysql database export dbpass=REPLACE_ME -# password for piwik web analytics admin -export piwik_pass=REPLACE_ME - -# git branch for mediawiki + extensions +# git branch for mediawiki + extensions. +# This guide has only been tested with 1_27. # branch names: https://git.wikimedia.org/branches/mediawiki%2Fcore.git export mw_branch=REL1_27 # customize these questions to something your contributors would know, -# is not super common / scriptable, and doesn't have the answer in the question +# and at least doesn't have the answer directly in the question captchaArray() { if ! grep -Fx '$localSettingsQuestyQuestions = array (' $mwc; then tee -a $mwc <<'EOF' @@ -102,31 +76,53 @@ $localSettingsQuestyQuestions = array ( "What does f in ofswiki.org stand for?" => "Free" ); EOF + fi } # As set by gui installer when choosing cc by sa. -export mw_RightsUrl="https://creativecommons.org/licenses/by-sa/4.0/"; -export mw_RightsText="Creative Commons Attribution-ShareAlike"; -export mw_RightsIcon="$wgResourceBasePath/resources/assets/licenses/cc-by-sa.png"; +export mw_RightsUrl='https://creativecommons.org/licenses/by-sa/4.0/' +export mw_RightsText='Creative Commons Attribution-ShareAlike' +export mw_RightsIcon='$wgScriptPath/resources/assets/licenses/cc-by-sa.png' -# domain name and install paths. These are examples, change as needed. -export mwfamily=ofswiki # short alphanumeric name for pywikibot -export mwdomain=ofswiki.org +export mwdomain=REPLACE_ME # domain name. for this site, it's ofswiki.org +# Alphanumeric site name for pywikibot. +# Here we use the domain minus the dots, which should work fine without changing. +export mwfamily=${mwdomain//./} +# install path for mediawiki. This should work fine. export mw=/var/www/$mwdomain/html/w # wiki sender address / wiki & wiki server contact email. # see email section for more info on email export mw_email="admin@$mwdomain" + +# Leave as is: +mwc="$mw/LocalSettings.php" -Here we define some small useful bash functions. This can be part of the same file. +== Download this page and run it == - -# for convenience, Mediawiki config file -mwc="$mw/LocalSettings.php" +This is an option to do automated setup. Optional code blocks are skipped (they have a bold warning just before them and a tag on the source block). The only important things left after running this are running the automated backup setup code on another machine. + +''' Requires manual step: inspect output file: /tmp/mw-setup, then run it''' + +start=' * *' +end=' *<\/source> *' +ruby <<'EOF' | sed -rn "/^$start$/,/^$end$/{s/^$start|$end$/# \0/;p} > /tmp/mw-setup" +require 'json' +puts JSON.parse(`curl 'https://ofswiki.org/w/api.php?\ +action=query&titles=Mediawiki_Setup_Guide&prop=revisions&rvprop=content&\ +format=json'`.chomp)['query']['pages'].values[0]['revisions'][0]['*'] +EOF +chmod +x /tmp/mw-setup + + +== Required Bash Functions == + +Here we define some small useful bash functions. This should be part of the same ~/mw_vars file if you are running the code step by step. + # identify if this is a debian based distro isdeb() { command -v apt &>/dev/null; } # tee unique. append each stdin line if it does not exist in the file @@ -139,57 +135,57 @@ teeu () { } # get and reset an extension/skin repository, and enable it -mw-ext () { mw-extra extensions $@; } -mw-skin() { mw-extra skins $@; } +mw-clone() { + local url=$1 + local original_pwd="$PWD" + local name + local re='[^/]*/[^/]*$' + [[ $url =~ $re ]] ||: + target=$mw/${BASH_REMATCH[0]} + if [[ ! -e $target/.git ]]; then + git clone $url $target + fi + if ! cd $target; then + echo "mw-ext error: failed cd $target"; + exit 1 + fi + git fetch + git checkout -qf origin/$mw_branch || git checkout -qf origin/master + git clean -xffd + cd "$original_pwd" -mw-extra() { - local type=$1 # extension or skin - shift +} +mw-ext () { local ext for ext in "$@"; do - local original_pwd="$PWD" - # it's ok that this fails if we already have it - url=https://git.wikimedia.org/git/mediawiki - target=$mw/$type/$ext - if [[ ! -e $target/.git ]]; then - git clone $url/$type/$ext.git $target - fi - if ! cd $target; then - echo "mw-ext error: failed cd $mw/extensions/$ext"; - exit 1 - fi - git fetch - git checkout -qf origin/$mw_branch || git checkout -qf origin/master - git clean -xffd - cd "$original_pwd" - case $type in - extensions) - if [[ -e $target/extension.json ]]; then - # new style extension. remove old style declaration - sed -i '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc - teeu $mwc </dev/null; then apache_user=www-data @@ -225,7 +221,7 @@ if isdeb; then # main reference: # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu apt-get update - apt-get install -y ImageMagick + apt-get install -y imagemagick php-mbstring if apt-get install -s mediawiki &>/dev/null; then # in debian wheezy time-frame distros, mediawiki was packaged. apt-get -y install php-apc mediawiki @@ -341,32 +337,6 @@ Upstream vs distro packages. Upstream is responsive, and it's distributed within Why use git over zip file releases? Mediawiki supports git usage through release branches which get post-release fixes. This means we can auto-update, get more granular fixes, easier to manage updates, and rollbacks. -== Get Piwik Files == - -Because it's really fun to look at your traffic stats. This is only needed for a production site. We get these files now so that we can take them into account when configuring apache in the next section. - -[http://piwik.org/docs/installation-maintenance/ Overall installation guide] - - -tmpdir="$(mktemp -d)" -cd $tmpdir -wget http://builds.piwik.org/piwik.zip -if isdeb; then apt-get -y install unzip; else yum -y install unzip; fi -unzip -q piwik.zip -# gui installer suggested command -if isdeb; then - chown -R www-data:www-data piwik -else - chown -R apache:apache piwik -fi -# remove any existing directory -rm -rf $mw/../analytics -mv piwik $mw/../analytics -cd $mw -rm -rf $tmpdir - - - == Configure Apache == Note, non-debian based installs: modify instructions below to use /etc/httpd/conf.d/$mwdomain.conf, and don't run a2ensite. @@ -418,11 +388,13 @@ RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L] # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L] EOF -find -L $(readlink -f $mw/..) -name .htaccess \ +find -L $(readlink -f $mw) -name .htaccess \ | while read line; do echo -e "\n $(< $line)\n"; done } | basic-https-conf/apache-site -r ${mw%/*} - $mwdomain +cd +rm -rf $temp Now mediawiki should load in your browser at $mwdomain . @@ -444,7 +416,7 @@ mw-skin Vector This section assumes we are redirecting www to a url without www. -[http://httpd.apache.org/docs/current/howto/htaccess.html Apache recommends] moving .htaccess rules into it's config for performance. So we look for .htaccess files from mediawiki and piwik and copy their contents into this config. In modern apache versions, we would have to explicitly set options like AllowOverride to allow .htaccess files to take effect. +[http://httpd.apache.org/docs/current/howto/htaccess.html Apache recommends] moving .htaccess rules into it's config for performance. So we look for .htaccess files from mediawiki and copy their contents into this config. In modern apache versions, we would have to explicitly set options like AllowOverride to allow .htaccess files to take effect. == Mediawiki Settings == @@ -495,7 +467,7 @@ EOF # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads # Increase from default of 2M to 100M. # This will at least allow high res pics etc. -php_ini=$(isdeb && echo /etc/php5/apache2/php.ini || echo /etc/php.ini) +php_ini=$(php -r 'echo(php_ini_loaded_file());') sed -i 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini if isdeb; then service apache2 restart @@ -609,7 +581,8 @@ if isdeb; then teeu $mwc <<'EOF' $wgExternalDiffEngine = 'wikidiff2'; EOF - ln -sf ../../mods-available/wikidiff2.ini /etc/php5/apache2/conf.d + dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d + ln -sf ../../mods-available/wikidiff2.ini $dir service apache2 restart fi @@ -631,7 +604,8 @@ else # todo, php5-curl equivalent on fedora yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml fi -ln -sf ../../mods-available/curl.ini /etc/php5/apache2/conf.d +dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d +ln -sf ../../mods-available/curl.ini $dir service apache2 restart cd $mw/extensions/Math/math; make # makes texvc @@ -970,6 +944,7 @@ To test a backup restore: # Change REPLACE_ME in the code below (as in the backup section so you get the right variables), # Execute the code on the backup machine. +'''Optional''' #!/bin/bash source ~/mw_vars @@ -1030,85 +1005,15 @@ My strategy is: # Backup the old database, restore it to the new wiki, run php maintenance/update.php. # If everything looks good, repeat and replace the old wiki with the new one. -== Setup Piwik == - -Note, - -# based on recommendation from install page -if ! dpkg -s php5-gd &>/dev/null; then - apt-get install -y php5-gd - service apache2 restart -fi - -# docs suggests using separate database user -mysql -u root -p$dbpass < - - -''' Manual Steps''' - -Unfortunately, Piwik does not document how to do a fully automated install, and although it certainly seems possible, my first glance at the config file after doing manual steps showed a lot of noise, so we are sticking with the manual steps for now. - -In a browser, navigate to your_domain/analytics. Follow these steps [http://piwik.org/docs/installation/#the-5-minute-piwik-installation the-5-minute-piwik-installation]. Use the user/db/pass from the previous section. - - -We are going to use an extension to handle the javaScript tracking tag part. - -Under settings -> user settings, click the cookie to exclude your current browser. - - -''' [[mediawikiwiki:Extension:Piwik Integration|Extension:Piwik Integration]] ''' - - -git clone https://github.com/DaSchTour/piwik-mediawiki-extension.git $mw/extensions/Piwik -mw-ext Piwik -teeu $mwc < - - -'''If this is not the first site you setup''' - -You will need to adjust the previous $wgPiwikIDSite variable. The correct value can be found in the Piwik javaScript tracking tag code, in the line that looks like _paq.push(['setSiteId', 2]); - -'''Updates''' - -Based on [http://piwik.org/docs/update/ piwik.org/docs/update]. Previously it's been seen to require manual steps, so it is not recommended to put in a cron script. - - -mkdir -p ${s%/*} -piwik_path=$mw/../analytics -cd "$(mktemp -d)" -wget -q http://builds.piwik.org/piwik.zip -unzip -q piwik.zip -rm -f piwik.zip -cp $piwik_path/config/config.ini.php piwik/config -cp -rf piwik/* $piwik_path -# prevent making an email out of the standard success response -$piwik_path/console core:update - - - -''' Skippable Notes ''' - -Piwik docs say use a subdir or a subdomain. googling finds a page which suggests using a subdomain, for more logical separation https://library.linode.com/web-applications/analytics/piwik/centos-5. However, mediawiki doesn't use the root dir of the web server, so the analytics folder can't mess things up by existing there, and it doesn't have any crazy virtualhost settings which might conflict, so there is no need. Plus we get ssl for free by sharing the same domain. - -The docs also say: "When you have finished configuring Piwik, you can set more restrictive permissions (ie. read only) to the “piwik/config/” folder. Piwik will always write its data inside the “piwik/tmp/” folder, so this is the only folder that requires write permissions." +== Stopping Spam == -But since the files are already owned by the web server user, and already can't be written by anyone else, it seems a bit of a pointless half measure since the owner can override any permissions. Enabling the .htaccess files seems more useful, which it doesn't even mention. +There is a balance between effective anti-spam measures and blocking/annoying contributors. Mediawiki documentation on how to combat spam, is not very good, but it has improved over time: [https://www.mediawiki.org/wiki/Manual:Combating_spam manual: Combating Spam]. It's possible for a spammer to quickly make thousands of edits, and there is no good documentation on purging lots of spam, so you should have a good strategy up front. My current strategy is 3 fold, and is limited to small/medium wiki's: -We could also set up a backup for piwik data, but traffic stat data is not that important so I haven't bothered. +* Find new spam quickly, revert it & ban the user. +** Watch, and get notified of changes on all primary content pages: Special:Preferences, Bottom of the page, set an email address, then turn on "Email me also for minor edits of pages and files." +** Use a rss/atom feed reader, and subscribe to recent changes across the wiki. Newer browsers have an rss feed subscribe button, you can click after going to Special:RecentChanges. If that is not available, you can construct the proper url based on [https://meta.wikimedia.org/wiki/Help:Recent_changes#Web_feed these instructions]. +* Require registration to edit, and a custom captcha question on registration. +* Install all non-user inhibiting anti-spam extensions / settings that take a reasonable amount of time to figure out. == Choosing Extensions == @@ -1180,6 +1085,7 @@ This means the code is compatible with gplv3. == todo list for this page == -* Evaluate any extensions bundled in 27 which I haven't checked out before. -* Visual editor +* Check if there are any new default extensions 1.27 which I haven't evaluated. +* Test for any new config values set by the 1.27 gui install method. +* Get Visual editor extension. * Don't require registration for edits diff --git a/mw-setup-script b/mw-setup-script index 7aca959..cae3ca4 100755 --- a/mw-setup-script +++ b/mw-setup-script @@ -2,9 +2,6 @@ # Copyright (C) 2016 Ian Kelling # This program is under GPL v. 3 or later, see # -# for convenience, Mediawiki config file -mwc="$mw/LocalSettings.php" - # identify if this is a debian based distro isdeb() { command -v apt &>/dev/null; } # tee unique. append each stdin line if it does not exist in the file @@ -17,57 +14,57 @@ teeu () { } # get and reset an extension/skin repository, and enable it -mw-ext () { mw-extra extensions $@; } -mw-skin() { mw-extra skins $@; } +mw-clone() { + local url=$1 + local original_pwd="$PWD" + local name + local re='[^/]*/[^/]*$' + [[ $url =~ $re ]] ||: + target=$mw/${BASH_REMATCH[0]} + if [[ ! -e $target/.git ]]; then + git clone $url $target + fi + if ! cd $target; then + echo "mw-ext error: failed cd $target"; + exit 1 + fi + git fetch + git checkout -qf origin/$mw_branch || git checkout -qf origin/master + git clean -xffd + cd "$original_pwd" -mw-extra() { - local type=$1 # extension or skin - shift +} +mw-ext () { local ext for ext in "$@"; do - local original_pwd="$PWD" - # it's ok that this fails if we already have it - url=https://git.wikimedia.org/git/mediawiki - target=$mw/$type/$ext - if [[ ! -e $target/.git ]]; then - git clone $url/$type/$ext.git $target - fi - if ! cd $target; then - echo "mw-ext error: failed cd $mw/extensions/$ext"; - exit 1 - fi - git fetch - git checkout -qf origin/$mw_branch || git checkout -qf origin/master - git clean -xffd - cd "$original_pwd" - case $type in - extensions) - if [[ -e $target/extension.json ]]; then - # new style extension. remove old style declaration - sed -i '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc - teeu $mwc </dev/null; then apache_user=www-data @@ -90,7 +87,7 @@ if isdeb; then # main reference: # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu apt-get update - apt-get install -y ImageMagick + apt-get install -y imagemagick php-mbstring if apt-get install -s mediawiki &>/dev/null; then # in debian wheezy time-frame distros, mediawiki was packaged. apt-get -y install php-apc mediawiki @@ -172,25 +169,6 @@ $wgGroupPermissions['*']['edit'] = false; # don't allow any account creation $wgGroupPermissions['*']['createaccount'] = false; EOF -# -# -tmpdir="$(mktemp -d)" -cd $tmpdir -wget http://builds.piwik.org/piwik.zip -if isdeb; then apt-get -y install unzip; else yum -y install unzip; fi -unzip -q piwik.zip -# gui installer suggested command -if isdeb; then - chown -R www-data:www-data piwik -else - chown -R apache:apache piwik -fi -# remove any existing directory -rm -rf $mw/../analytics -mv piwik $mw/../analytics -cd $mw -rm -rf $tmpdir - # # temp=$(mktemp -d) @@ -211,11 +189,13 @@ RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L] # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L] EOF -find -L $(readlink -f $mw/..) -name .htaccess \ +find -L $(readlink -f $mw) -name .htaccess \ | while read line; do echo -e "\n $(< $line)\n"; done } | basic-https-conf/apache-site -r ${mw%/*} - $mwdomain +cd +rm -rf $temp # # dd of=$mw/../robots.txt <<'EOF' @@ -267,7 +247,7 @@ EOF # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads # Increase from default of 2M to 100M. # This will at least allow high res pics etc. -php_ini=$(isdeb && echo /etc/php5/apache2/php.ini || echo /etc/php.ini) +php_ini=$(php -r 'echo(php_ini_loaded_file());') sed -i 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini if isdeb; then service apache2 restart @@ -314,7 +294,8 @@ if isdeb; then teeu $mwc <<'EOF' $wgExternalDiffEngine = 'wikidiff2'; EOF - ln -sf ../../mods-available/wikidiff2.ini /etc/php5/apache2/conf.d + dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d + ln -sf ../../mods-available/wikidiff2.ini $dir service apache2 restart fi # @@ -332,7 +313,8 @@ else # todo, php5-curl equivalent on fedora yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml fi -ln -sf ../../mods-available/curl.ini /etc/php5/apache2/conf.d +dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d +ln -sf ../../mods-available/curl.ini $dir service apache2 restart cd $mw/extensions/Math/math; make # makes texvc @@ -574,30 +556,3 @@ php $mw/maintenance/update.php -q EOF chmod +x $s # -# -# based on recommendation from install page -if ! dpkg -s php5-gd &>/dev/null; then - apt-get install -y php5-gd - service apache2 restart -fi - -# docs suggests using separate database user -mysql -u root -p$dbpass < -# -git clone https://github.com/DaSchTour/piwik-mediawiki-extension.git $mw/extensions/Piwik -mw-ext Piwik -teeu $mwc < -- 2.30.2