iankelling.org Git - mediawiki-setup/blob - mw-setup-script

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3 # This program is under GPL v. 3 or later, see <http://www.gnu.org/licenses/>
   4 set -x
   5 # <source lang="bash">
   6 apt-get install --install-suggests jessie-backports certbot
   7 # </source>
   8 # <source lang="bash">
   9 # identify if this is a debian based distro
  10 isdeb() { command -v apt &>/dev/null; }
  11 # tee unique. append each stdin line if it does not exist in the file
  12 teeu () {
  13     local MAPFILE
  14     mapfile -t
  15     for line in "${MAPFILE[@]}"; do
  16         grep -xFq "$line" "$1" &>/dev/null || tee -a "$1" <<<"$line"
  17     done
  18 }
  19
  20 # get and reset an extension/skin repository, and enable it
  21 mw-clone() {
  22     local url=$1
  23     local original_pwd="$PWD"
  24     local name
  25     local re='[^/]*/[^/]*$' # last 2 parts of path
  26     [[ $url =~ $re ]] ||:
  27     target=$mw/${BASH_REMATCH[0]}
  28     if [[ ! -e $target/.git ]]; then
  29         git clone $url $target
  30     fi
  31     if ! cd $target; then
  32         echo "mw-ext error: failed cd $target";
  33         exit 1
  34     fi
  35     git fetch
  36     git checkout -qf origin/$mw_branch || git checkout -qf origin/master
  37     git clean -xffd
  38     cd "$original_pwd"
  39
  40 }
  41 mw-ext () {
  42     local ext
  43     for ext; do
  44         mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/$ext
  45         if [[ -e $mw/extensions/$ext/extension.json ]]; then
  46             # new style extension
  47             teeu $mwc <<EOF
  48 wfLoadExtension( '$ext' );
  49 EOF
  50         else
  51             teeu $mwc <<EOF
  52 require_once( "\$IP/extensions/$ext/$ext.php" );
  53 EOF
  54         fi
  55     done
  56     # --quick is quicker than default flags,
  57     # but still add a sleep to make sure everything works right
  58     sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
  59 }
  60 mw-skin() {
  61     local skin=$1
  62     mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/skins/$skin
  63     sed -i --follow-symlinks '/^wfLoadSkin/d' $mwc
  64     sed -i --follow-symlinks '/^\$wgDefaultSkin/d' $mwc
  65     teeu $mwc <<EOF
  66 \$wgDefaultSkin = "${skin,,*}";
  67 wfLoadSkin( '$skin' );
  68 EOF
  69     sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
  70 }
  71
  72 if command -v apt &>/dev/null; then
  73     apache_user=www-data
  74 else
  75     apache_user=apache
  76 fi
  77
  78 # </source>
  79 # <source lang="bash">
  80 # From here on out, exit if a command fails.
  81 # This will prevent us from not noticing an important failure.
  82 # We recommend setting this for the entire installation session.
  83 # If you are running commands interactively, it might be best to
  84 # put it in your ~/.bashrc temporarily.
  85 set -eE -o pipefail
  86 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
  87 source ~/mw_vars
  88
  89 if isdeb; then
  90     # main reference:
  91     # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu
  92     apt-get update
  93     DEBIAN_FRONTEND=noninteractive apt-get install -y imagemagick curl
  94     if apt-get install -s mediawiki &>/dev/null; then
  95         # mediawiki is packaged in jessie backports.
  96         DEBIAN_FRONTEND=noninteractive apt-get -y install php5-apcu mediawiki
  97     else
  98         # https://www.mediawiki.org/wiki/Manual:Installation_requirements
  99         if apt-get install -s php7.0 &>/dev/null; then
 100             # note, 7.0 is untested by the editor here, since it's not
 101             # available in debian 8. it's listed as supported
 102             # in the mediawiki page.
 103             # noninteractive to avoid mysql password prompt.
 104             DEBIAN_FRONTEND=noninteractive apt-get install -y apache2  \
 105                            default-mysql-server \
 106                            php7.0 php7.0-mysql libapache2-mod-php7.0 php7.0-xml \
 107                            php7.0-apcu php7.0-mbstring
 108         else
 109             # note: mbstring is recommended, but it's not available for php5 in
 110             # debian jessie.
 111             DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 \
 112                            default-mysql-server \
 113                            php5 php5-mysql libapache2-mod-php5 php5-apcu
 114         fi
 115     fi
 116     service apache2 restart
 117 else
 118     # note
 119     # fedora deps are missing a database, so some is translated from debian packages
 120     yum -y install mediawiki ImageMagick php-mysqlnd php-pecl-apcu mariadb-server
 121
 122     systemctl restart mariadb.service
 123     systemctl enable mariadb.service
 124     systemctl enable httpd.service
 125     systemctl restart httpd.service
 126 fi
 127
 128
 129 # skip if we already set the root pass
 130 if ! echo exit|mysql -uroot "-p$dbpass"; then
 131     # Note: we set a root password here, but in debian 9+, it is ignored;
 132     # only the local user root can login, and a password is silently ignored.
 133     # We answer these interactive prompts:
 134     # Enter current password for root (enter for none):
 135     # Set root password? [Y/n]
 136     # New password:
 137     # Re-enter new password:
 138     # Remove anonymous users? [Y/n]
 139     # Disallow root login remotely? [Y/n]
 140     # Remove test database and access to it? [Y/n]
 141     # Reload privilege tables now? [Y/n]
 142     # Note, I had 1 less newline at the start when doing ubuntu 14.04,
 143     # compared to debian 8, so can't say this is especially portable.
 144     echo -e "\n\n$dbpass\n$dbpass\n\n\n\n\n" | mysql_secure_installation
 145 fi
 146 mysql -uroot "-p$dbpass" <<EOF
 147 GRANT ALL PRIVILEGES ON my_wiki.* TO 'wikiuser'@'localhost' IDENTIFIED BY '$dbpass';
 148 EOF
 149 # </source>
 150 # <source lang="bash">
 151 mkdir -p $mw
 152 cd $mw
 153 # this will just fail if it already exists which is fine
 154 if [[ ! -e .git ]]; then
 155     git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
 156 fi
 157 # to see available branches: https://www.mediawiki.org/wiki/Version_lifecycle
 158 # and
 159 # git branch -r
 160 git checkout -f origin/$mw_branch
 161 git clean -ffxd
 162 # apply librejs patch
 163 curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-1.28-librejs.patch;hb=HEAD" | patch -r - -N -p1
 164 # Get the php libraries wmf uses. Based on:
 165 # https://www.mediawiki.org/wiki/Download_from_Git#Fetch_external_libraries
 166 if [[ ! -e vendor/.git ]]; then
 167     git clone  https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
 168 fi
 169 cd vendor
 170 git checkout -f origin/$mw_branch
 171 cd ..
 172
 173 # Drop any previous database which may have been installed while testing.
 174 # If upgrading, we should have a db backup which will get restored.
 175 # https://www.mediawiki.org/wiki/Manual:Upgrading
 176 mysql -uroot "-p$dbpass" <<'EOF' ||:
 177 drop database my_wiki;
 178 exit
 179 EOF
 180 php $mw/maintenance/install.php --pass $wikipass --scriptpath /w \
 181     --dbuser wikiuser --dbpass $dbpass "$mwdescription" "$wikiuser"
 182 teeu $mwc <<'EOF'
 183 # lock down the wiki to only the initial owner until anti-spam measures are put in place
 184 # limit edits to registered users
 185 $wgGroupPermissions['*']['edit'] = false;
 186 # don't allow any account creation
 187 $wgGroupPermissions['*']['createaccount'] = false;
 188 EOF
 189 # </source>
 190 # <source lang="bash">
 191 temp=$(mktemp -d)
 192 cd $temp
 193 git_site=https://iankelling.org/git
 194 l=$mw/../../logs
 195 mkdir -p $l
 196
 197 git clone $git_site/basic-https-conf
 198 basic-https-conf/web-conf -r ${mw%/*} - apache2 $mwdomain <<EOF
 199 ServerAdmin $mw_email
 200 RewriteEngine On
 201 # make the site's root url go to our main page
 202 RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
 203 # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
 204 RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
 205 EOF
 206 find -L $(readlink -f $mw) -name .htaccess \
 207     | while read line; do
 208     echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
 209 done
 210 cd
 211 rm -rf $temp
 212 # </source>
 213 # <source lang="bash">
 214 dd of=$mw/../robots.txt <<'EOF'
 215         User-agent: *
 216 Disallow: /w/
 217         User-agent: ia_archiver
 218 Allow: /*&action=raw
 219 EOF
 220 mw-skin Vector
 221 # </source>
 222 # <source lang="bash">
 223 teeu $mwc<<EOF
 224 \$wgServer = "https://$mwdomain";
 225 \$wgDBserver = "localhost";
 226 \$wgRightsUrl = "$mw_RightsUrl";
 227 \$wgRightsText = "$mw_RightsText";
 228 \$wgRightsIcon = "$mw_RightsIcon";
 229 EOF
 230 # </source>
 231 # <source lang="bash">
 232 teeu $mwc<<EOF
 233 \$wgPasswordSender = "$mw_email";
 234 \$wgEmergencyContact = "$mw_email";
 235 \$wgEnotifUserTalk = true; # UPO
 236 \$wgEnotifWatchlist = true; # UPO
 237 \$wgMainCacheType = CACHE_ACCEL;
 238 \$wgEnableUploads = true;
 239 \$wgUseInstantCommons = true;
 240 \$wgPingback = true;
 241 EOF
 242 # </source>
 243 # <source lang="bash">
 244 teeu $mwc <<'EOF'
 245 # from https://www.mediawiki.org/wiki/Manual:Short_URL
 246 $wgArticlePath = "/wiki/$1";
 247
 248 # https://www.mediawiki.org/wiki/Manual:Combating_spam
 249 # check that url if our precautions don't work
 250 # not using nofollow is good practice, as long as we avoid spam.
 251 $wgNoFollowLinks = false;
 252 # Allow user customization.
 253 $wgAllowUserCss = true;
 254 # use imagemagick over GD
 255 $wgUseImageMagick = true;
 256 # manual says this is not production ready, I think that is mostly
 257 # because they are using MobileFrontend extension instead, which gives
 258 # an even cleaner more minimal view, I plan to try setting it up
 259 # sometime but this seems like a very nice improvement for now.
 260 $wgVectorResponsive = true;
 261 EOF
 262
 263
 264 # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads
 265 # Increase from default of 2M to 100M.
 266 # This will at least allow high res pics etc.
 267 php_ini=$(php -r 'echo(php_ini_loaded_file());')
 268 sed -i --follow-symlinks 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini
 269 if isdeb; then
 270     service apache2 restart
 271 else
 272     systemctl restart httpd.service
 273 fi
 274
 275 # if you were to install as a normal user, you would need this for images
 276 # sudo usermod -aG $apache_user $USER
 277
 278 # this doesn't propogate right away
 279 chgrp -R $apache_user $mw/images
 280 chmod -R g+w $mw/images
 281 # </source>
 282 # <source lang="bash">
 283 teeu $mwc <<'EOF'
 284 $wgLogo = null;
 285 #$wgFooterIcons = null;
 286 EOF
 287 # Make the toolbox go into the drop down.
 288 cd $mw/skins/Vector
 289 if ! git remote show ian-kelling &>/dev/null; then
 290     git remote add ian-kelling https://iankelling.org/git/forks/Vector
 291 fi
 292 git fetch ian-kelling
 293 git checkout ian-kelling/${mw_branch}-toolbox-in-dropdown
 294 # </source>
 295 # <source lang="bash">
 296 mw-ext Cite CiteThisPage CheckUser CSS Echo Gadgets ImageMap Interwiki News \
 297        Nuke ParserFunctions Poem Renameuser SyntaxHighlight_GeSHi Variables
 298 # </source>
 299 # <source lang="bash">
 300 mw-ext AntiSpoof
 301 # recommended setup script to account for existing users
 302 sudo -u $apache_user php $mw/extensions/AntiSpoof/maintenance/batchAntiSpoof.php
 303 # </source>
 304 # <source lang="bash">
 305 if isdeb; then
 306     apt-get -y install php-wikidiff2
 307     teeu $mwc <<'EOF'
 308 $wgExternalDiffEngine = 'wikidiff2';
 309 EOF
 310     dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d
 311     ln -sf ../../mods-available/wikidiff2.ini $dir
 312     service apache2 restart
 313 fi
 314 # </source>
 315 # <source lang="bash">
 316 mw-ext Math
 317 # php5-curl according to Math readme
 318 if isdeb; then
 319     curl_pkg=php7.0-curl
 320     if ! apt-get -s install $curl_pkg &>/dev/null; then
 321         curl_pkg=php5-curl
 322     fi
 323     apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base \
 324             ghostscript imagemagick ocaml $curl_pkg make
 325 else
 326     # todo, php5-curl equivalent on fedora
 327     yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml
 328 fi
 329 service apache2 restart
 330
 331 cd $mw/extensions/Math/math; make # makes texvc
 332 cd $mw/extensions/Math/texvccheck; make
 333
 334 teeu $mwc <<'EOF'
 335 # Enable MathJax as rendering option
 336 $wgUseMathJax = true;
 337 # Enable LaTeXML as rendering option
 338 $wgMathValidModes[] = 'latexml';
 339 # Set LaTeXML as default rendering option, because it is nicest
 340 $wgDefaultUserOptions['math'] = 'latexml';
 341 EOF
 342 # </source>
 343 # <source lang="bash">
 344 mw-ext SpamBlacklist
 345 if ! grep -F '$wgSpamBlacklistFiles = array(' $mwc &>/dev/null; then
 346     tee -a $mwc <<'EOF'
 347 $wgEnableDnsBlacklist = true;
 348 $wgDnsBlacklistUrls = array( 'xbl.spamhaus.org', 'dnsbl.tornevall.org' );
 349
 350 ini_set( 'pcre.backtrack_limit', '10M' );
 351 $wgSpamBlacklistFiles = array(
 352    "[[m:Spam blacklist]]",
 353    "http://en.wikipedia.org/wiki/MediaWiki:Spam-blacklist"
 354 );
 355 EOF
 356 fi
 357 # </source>
 358 # <source lang="bash">
 359 mw-ext TitleBlacklist
 360 if ! grep -F '$wgTitleBlacklistSources = array(' $mwc &>/dev/null; then
 361     tee -a $mwc <<'EOF'
 362 $wgTitleBlacklistSources = array(
 363     array(
 364          'type' => 'local',
 365          'src'  => 'MediaWiki:Titleblacklist',
 366     ),
 367     array(
 368          'type' => 'url',
 369          'src'  => 'http://meta.wikimedia.org/w/index.php?title=Title_blacklist&action=raw',
 370     ),
 371 );
 372 EOF
 373 fi
 374 # </source>
 375 # <source lang="bash">
 376 mw-ext WikiEditor
 377 teeu $mwc <<'EOF'
 378 # Enable Wikieditor by default
 379 $wgDefaultUserOptions['usebetatoolbar'] = 1;
 380 $wgDefaultUserOptions['usebetatoolbar-cgd'] = 1;
 381
 382 # Display the Preview and Changes tabs
 383 $wgDefaultUserOptions['wikieditor-preview'] = 1;
 384 EOF
 385 # </source>
 386 # <source lang="bash">
 387 mw-ext CategoryTree
 388 teeu $mwc <<'EOF'
 389 # Mediawiki setting dependency for CategoryTree
 390 $wgUseAjax = true;
 391 EOF
 392 # </source>
 393 # <source lang="bash">
 394 mw-ext AbuseFilter
 395 teeu $mwc<<'EOF'
 396 $wgGroupPermissions['sysop']['abusefilter-modify'] = true;
 397 $wgGroupPermissions['*']['abusefilter-log-detail'] = true;
 398 $wgGroupPermissions['*']['abusefilter-view'] = true;
 399 $wgGroupPermissions['*']['abusefilter-log'] = true;
 400 $wgGroupPermissions['sysop']['abusefilter-private'] = true;
 401 $wgGroupPermissions['sysop']['abusefilter-modify-restricted'] = true;
 402 $wgGroupPermissions['sysop']['abusefilter-revert'] = true;
 403 EOF
 404 # </source>
 405 # <source lang="bash">
 406 mw-ext ConfirmEdit
 407 captchaArray
 408 teeu $mwc <<'EOF'
 409 wfLoadExtension( 'ConfirmEdit/QuestyCaptcha' );
 410 $wgCaptchaClass = 'QuestyCaptcha';
 411 # only captcha on registration
 412 $wgGroupPermissions['user'         ]['skipcaptcha'] = true;
 413 $wgGroupPermissions['autoconfirmed']['skipcaptcha'] = true;
 414 EOF
 415 if ! grep -Fx 'foreach ( $localSettingsQuestyQuestions as $key => $value ) {' $mwc; then
 416     tee -a $mwc <<'EOF'
 417 foreach ( $localSettingsQuestyQuestions as $key => $value ) {
 418         $wgCaptchaQuestions[] = array( 'question' => $key, 'answer' => $value );
 419 }
 420 EOF
 421 fi
 422 # </source>
 423 # <source lang="bash">
 424 sed -i --follow-symlinks "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
 425 # </source>
 426 # <source lang="bash">
 427 # get repo
 428 if [[ ! -e ~/pywikibot/.git ]]; then
 429     git clone --recursive \
 430         https://gerrit.wikimedia.org/r/pywikibot/core.git ~/pywikibot
 431 fi
 432 cd ~/pywikibot
 433 #updating
 434 git pull --all
 435 git submodule update
 436 # </source>
 437 # <source lang="bash">
 438 cd $HOME/pywikibot
 439 dd of=user-config.py <<EOF
 440 mylang = 'en'
 441 usernames["$mwfamily"]['en'] = u'$wikiuser'
 442 family = "$mwfamily"
 443 console_encoding = 'utf-8'
 444 password_file = "secretsfile"
 445 EOF
 446
 447 dd of=secretsfile <<EOF
 448 ("$wikiuser", "$wikipass")
 449 EOF
 450
 451 # it won't overrwrite an existing file. Remove if if one exists
 452 rm -f  pywikibot/families/${mwfamily}_family.py
 453 if isdeb; then
 454     apt-get install -y python-requests
 455 else
 456     yum -y install python-requests
 457 fi
 458
 459 python generate_family_file.py https://$mwdomain/wiki/Main_Page "$mwfamily"
 460
 461 # Note, this needed only for ssl site
 462 tee -a pywikibot/families/${mwfamily}_family.py<<'EOF'
 463     def protocol(self, code):
 464         return 'https'
 465 EOF
 466 # </source>
 467 # <source lang="bash">
 468 cd "$HOME/pywikibot"
 469
 470 dd of=scripts/${mwfamily}_setup.py<<EOF
 471 import pywikibot
 472 import time
 473 import sys
 474 site = pywikibot.Site()
 475 def x(p, t=""):
 476    page = pywikibot.Page(site, p)
 477    page.text = t
 478    #force is for some anti-bot thing, not necessary in my testing, but might as well include it
 479    page.save(force=True)
 480
 481 # Small/medium noncommercial wiki should be fine with no privacy policy
 482 # based on https://www.mediawiki.org/wiki/Manual:Footer
 483 x("MediaWiki:Privacy")
 484
 485 # licenses for uploads. Modified from the mediawiki's wiki
 486 x("MediaWiki:Licenses", u"""* Same as this wiki's text (preferred)
 487 ** CC BY-SA or GFDL| Creative Commons Attribution ShareAlike or GNU Free Documentation License
 488 * Others:
 489 ** Unknown_copyright|I don't know exactly
 490 ** PD|PD: public domain
 491 ** CC BY|Creative Commons Attribution
 492 ** CC BY-SA|Creative Commons Attribution ShareAlike
 493 ** GFDL|GFDL: GNU Free Documentation License
 494 ** GPL|GPL: GNU General Public License
 495 ** LGPL|LGPL: GNU Lesser General Public License""")
 496 x("MediaWiki:Copyright", '$mw_license')
 497 x("MediaWiki:Mainpage-description", "$mwdescription")
 498
 499
 500
 501 # The rest of the settings are for the site style
 502
 503 # Remove various clutter
 504 x("MediaWiki:Lastmodifiedat")
 505 x("MediaWiki:Disclaimers")
 506 x("MediaWiki:Viewcount")
 507 x("MediaWiki:Aboutsite")
 508 # remove these lines from sidebar
 509 # ** recentchanges-url|recentchanges
 510 # ** randompage-url|randompage
 511 # ** helppage|help
 512 x("MediaWiki:Sidebar", """* navigation
 513 ** mainpage|mainpage-description
 514 * SEARCH
 515 * TOOLBOX
 516 * LANGUAGES""")
 517
 518 # remove side panel
 519 # helpfull doc: https://www.mediawiki.org/wiki/Manual:Interface/Sidebar
 520 x("mediawiki:Common.css", """/* adjust sidebar to just be home link and up top  */
 521 /* adjust sidebar to just be home link and up top  */
 522 /* panel width increased to fit full wiki name. */
 523 /* selectors other than final id are for increasing priority of rule */
 524 div#mw-panel { top: 10px; padding-top: 0em; width: 20em }
 525 div#footer, #mw-head-base, div#content { margin-left: 1em; }
 526 #left-navigation { margin-left: 1em; }
 527
 528
 529 /* logo, and toolbar hidden */
 530 #p-logo, div#mw-navigation div#mw-panel #p-tb {
 531    display:none;
 532 }
 533
 534 div#mw-content-text {
 535     max-width: 720px;
 536 }
 537 """)
 538 EOF
 539
 540 # this can spam a warning, so uniq it
 541 python pwb.py ${mwfamily}_setup |& uniq
 542 # </source>
 543 # <source lang="bash">
 544 s=/etc/cron.daily/mediawiki_update
 545 dd of=$s<<'EOF'
 546 #!/bin/bash
 547 source ~/mw_vars
 548 update() {
 549     dir=$1
 550     cd $mw
 551     [[ -d $dir ]] || return 1
 552     cd $dir
 553     branch=$(git describe --all)
 554     branch=${branch#remotes/}
 555     git fetch --all -q
 556     new_head=$(git rev-parse $branch)
 557     log=$(git log HEAD..$new_head)
 558     if [[ ! $log ]]; then
 559         return 1
 560     fi
 561     pwd
 562     echo "$log"
 563     git checkout -qf $new_head
 564     cd $mw
 565     return 0
 566 }
 567 for dir in extensions/* skins/* vendor; do
 568     update "$dir" ||:
 569 done
 570 if update .; then
 571     curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-1.28-librejs.patch;hb=HEAD" | patch -r - -N -p1
 572 fi
 573 php $mw/maintenance/update.php -q --quick
 574 EOF
 575
 576 # </source>