iankelling.org Git - mediawiki-setup/blob - mw-setup-script

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3 # This program is under GPL v. 3 or later, see <http://www.gnu.org/licenses/>
   4 # <source lang="bash">
   5 # identify if this is a debian based distro
   6 isdeb() { command -v apt &>/dev/null; }
   7 # tee unique. append each stdin line if it does not exist in the file
   8 teeu () {
   9     local MAPFILE
  10     mapfile -t
  11     for line in "${MAPFILE[@]}"; do
  12         grep -xFq "$line" "$1" &>/dev/null || tee -a "$1" <<<"$line"
  13     done
  14 }
  15
  16 # get and reset an extension/skin repository, and enable it
  17 mw-clone() {
  18     local url=$1
  19     local original_pwd="$PWD"
  20     local name
  21     local re='[^/]*/[^/]*$'
  22     [[ $url =~ $re ]] ||:
  23     target=$mw/${BASH_REMATCH[0]}
  24     if [[ ! -e $target/.git ]]; then
  25         git clone $url $target
  26     fi
  27     if ! cd $target; then
  28         echo "mw-ext error: failed cd $target";
  29         exit 1
  30     fi
  31     git fetch
  32     git checkout -qf origin/$mw_branch || git checkout -qf origin/master
  33     git clean -xffd
  34     cd "$original_pwd"
  35
  36 }
  37 mw-ext () {
  38     local ext
  39     for ext in "$@"; do
  40         mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/$ext
  41         if [[ -e $mw/ext/$ext/extension.json ]]; then
  42             # new style extension. remove old style declaration
  43             sed -i --follow-symlinks '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc
  44             teeu $mwc <<EOF
  45 wfLoadExtension( '$ext' );
  46 EOF
  47         else
  48             teeu $mwc <<EOF
  49 require_once( "\$IP/extensions/$ext/$ext.php" );
  50 EOF
  51         fi
  52     done
  53     # --quick is quicker than default flags,
  54     # but still add a sleep to make sure everything works right
  55     sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
  56 }
  57 mw-skin() {
  58     local skin=$1
  59     mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/skins/$skin
  60     sed -i --follow-symlinks '/^wfLoadSkin/d' $mwc
  61     sed -i --follow-symlinks '/^\$wgDefaultSkin/d' $mwc
  62     teeu $mwc <<EOF
  63 \$wgDefaultSkin = "${skin,,*}";
  64 wfLoadSkin( '$skin' );
  65 EOF
  66     sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
  67 }
  68
  69 if command -v apt &>/dev/null; then
  70     apache_user=www-data
  71 else
  72     apache_user=apache
  73 fi
  74
  75 # </source>
  76 # <source lang="bash">
  77 # From here on out, exit if a command fails.
  78 # This will prevent us from not noticing an important failure.
  79 # We recommend setting this for the entire installation session.
  80 # If you are running commands interactively, it might be best to
  81 # put it in your ~/.bashrc temporarily.
  82 set -eE -o pipefail
  83 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
  84 source ~/mw_vars
  85
  86 if isdeb; then
  87     # main reference:
  88     # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu
  89     apt-get update
  90     apt-get install -y imagemagick
  91     if apt-get install -s mediawiki &>/dev/null; then
  92         # in debian wheezy time-frame distros, mediawiki was packaged.
  93         apt-get -y install php-apc mediawiki php-mbstring
  94     else
  95         # https://www.mediawiki.org/wiki/Manual:Installation_requirements
  96         if apt-get install -s php7.0 &>/dev/null; then
  97             # note, 7.0 is untested by the editor here, since it's not
  98             # available in debian 8. it's listed as supported
  99             # in the mediawiki page.
 100             # noninteractive to avoid mysql password prompt.
 101             DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 mysql-server \
 102                            php7.0 php7.0-mysql libapache2-mod-php7.0 php7.0-xml \
 103                            php7.0-apcu php7.0-mbstring
 104         else
 105             # note: mbstring is recommended, but it's not available for php5 in
 106             # debian jessie.
 107             DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 mysql-server \
 108                            php5 php5-mysql libapache2-mod-php5 php5-apcu
 109         fi
 110     fi
 111     service apache2 restart
 112 else
 113     # note
 114     # fedora deps are missing a database, so some is translated from debian packages
 115     yum -y install mediawiki ImageMagick php-mysqlnd php-pecl-apcu mariadb-server
 116
 117     systemctl restart mariadb.service
 118     systemctl enable mariadb.service
 119     systemctl enable httpd.service
 120     systemctl restart httpd.service
 121 fi
 122
 123
 124 # slightly different depending on if we already set the root pass
 125 if echo exit|mysql -u root -p"$dbpass"; then
 126     # answer interactive prompts:
 127     # mysql root pass, change pass? no, remove anon users? (default, yes)
 128     # disallow remote root (default, yes), reload? (default, yes)
 129     echo -e "$dbpass\nn\n\n\n\n" | mysql_secure_installation
 130 else
 131     # I had 1 less newline at the start when doing ubuntu 14.04,
 132     # compared to debian 8, so can't say this is especially portable.
 133     # It won't hurt if it fails.
 134     echo -e "\n\n$dbpass\n$dbpass\n\n\n\n\n" | mysql_secure_installation
 135 fi
 136 # </source>
 137 # <source lang="bash">
 138 mkdir -p $mw
 139 cd $mw
 140 # this will just fail if it already exists which is fine
 141 if [[ ! -e .git ]]; then
 142     git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
 143 fi
 144 # to see available branches: https://www.mediawiki.org/wiki/Version_lifecycle
 145 # and
 146 # git branch -r
 147 git checkout -f origin/$mw_branch
 148 git clean -ffxd
 149 # Get the php libraries wmf uses. Based on:
 150 # https://www.mediawiki.org/wiki/Download_from_Git#Fetch_external_libraries
 151 if [[ ! -e vendor/.git ]]; then
 152     git clone  https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
 153 fi
 154 cd vendor
 155 git checkout -f origin/$mw_branch
 156 cd ..
 157
 158 # Drop any previous database which may have been installed while testing.
 159 # If upgrading, we should have a db backup which will get restored.
 160 # https://www.mediawiki.org/wiki/Manual:Upgrading
 161 mysql -u root -p$dbpass <<'EOF' ||:
 162 drop database my_wiki;
 163 exit
 164 EOF
 165 php $mw/maintenance/install.php --pass $wikipass --scriptpath /w \
 166     --dbuser root --dbpass $dbpass "$mwdescription" "$wikiuser"
 167 teeu $mwc <<'EOF'
 168 # lock down the wiki to only the initial owner until anti-spam measures are put in place
 169 # limit edits to registered users
 170 $wgGroupPermissions['*']['edit'] = false;
 171 # don't allow any account creation
 172 $wgGroupPermissions['*']['createaccount'] = false;
 173 EOF
 174 # </source>
 175 # <source lang="bash">
 176 temp=$(mktemp -d)
 177 cd $temp
 178 git_site=https://iankelling.org/git
 179 git clone $git_site/acme-tiny-wrapper
 180 l=$mw/../../logs
 181 mkdir -p $l
 182
 183 acme-tiny-wrapper/acme-tiny-wrapper -t $mwdomain
 184
 185 git clone $git_site/basic-https-conf
 186 { cat <<EOF
 187 ServerAdmin $mw_email
 188 RewriteEngine On
 189 # make the site's root url go to our main page
 190 RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
 191 # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
 192 RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
 193 EOF
 194 find -L $(readlink -f $mw) -name .htaccess \
 195     | while read line; do
 196     echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
 197 done
 198 } | basic-https-conf/apache-site -r ${mw%/*} - $mwdomain
 199 cd
 200 rm -rf $temp
 201 # </source>
 202 # <source lang="bash">
 203 dd of=$mw/../robots.txt <<'EOF'
 204         User-agent: *
 205 Disallow: /w/
 206         User-agent: ia_archiver
 207 Allow: /*&action=raw
 208 EOF
 209 mw-skin Vector
 210 # </source>
 211 # <source lang="bash">
 212 teeu $mwc<<EOF
 213 \$wgServer = "https://$mwdomain";
 214 \$wgDBserver = "localhost";
 215 \$wgRightsUrl = "$mw_RightsUrl";
 216 \$wgRightsText = "$mw_RightsText";
 217 \$wgRightsIcon = "$mw_RightsIcon";
 218 EOF
 219 # </source>
 220 # <source lang="bash">
 221 teeu $mwc<<EOF
 222 \$wgPasswordSender = "$mw_email";
 223 \$wgEmergencyContact = "$mw_email";
 224 \$wgEnotifUserTalk = true; # UPO
 225 \$wgEnotifWatchlist = true; # UPO
 226 \$wgMainCacheType = CACHE_ACCEL;
 227 \$wgEnableUploads = true;
 228 \$wgUseInstantCommons = true;
 229 EOF
 230 # </source>
 231 # <source lang="bash">
 232 teeu $mwc <<'EOF'
 233 # from https://www.mediawiki.org/wiki/Manual:Short_URL
 234 $wgArticlePath = "/wiki/$1";
 235
 236 # https://www.mediawiki.org/wiki/Manual:Combating_spam
 237 # check that url if our precautions don't work
 238 # not using nofollow is good practice, as long as we avoid spam.
 239 $wgNoFollowLinks = false;
 240 # Allow user customization.
 241 $wgAllowUserJs = true;
 242 $wgAllowUserCss = true;
 243
 244 # use imagemagick over GD
 245 $wgUseImageMagick = true;
 246 EOF
 247
 248
 249 # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads
 250 # Increase from default of 2M to 100M.
 251 # This will at least allow high res pics etc.
 252 php_ini=$(php -r 'echo(php_ini_loaded_file());')
 253 sed -i --follow-symlinks 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini
 254 if isdeb; then
 255     service apache2 restart
 256 else
 257     systemctl restart httpd.service
 258 fi
 259
 260 # if you were to install as a normal user, you would need this for images
 261 # sudo usermod -aG $apache_user $USER
 262
 263 # this doesn't propogate right away
 264 chgrp -R $apache_user $mw/images
 265 chmod -R g+w $mw/images
 266 # </source>
 267 # <source lang="bash">
 268 teeu $mwc <<'EOF'
 269 $wgLogo = null;
 270 #$wgFooterIcons = null;
 271 EOF
 272 # Make the toolbox go into the drop down.
 273 cd $mw/skins/Vector
 274 if ! git remote show ian-kelling &>/dev/null; then
 275     git remote add ian-kelling https://iankelling.org/git/forks/Vector
 276 fi
 277 git fetch ian-kelling
 278 git checkout ian-kelling/REL1_27-toolbox-in-dropdown
 279 # </source>
 280 # <source lang="bash">
 281 mw-ext Cite CiteThisPage CSS Echo Gadgets ImageMap Interwiki News \
 282        Nuke ParserFunctions Poem SyntaxHighlight_GeSHi Variables
 283 # </source>
 284 # <source lang="bash">
 285 mw-ext AntiSpoof
 286 # recommended setup script to account for existing users
 287 sudo -u $apache_user php $mw/extensions/AntiSpoof/maintenance/batchAntiSpoof.php
 288 # </source>
 289 # <source lang="bash">
 290 mw-ext CheckUser
 291 sudo -u $apache_user php $mw/extensions/CheckUser/install.php; sleep 1
 292 # </source>
 293 # <source lang="bash">
 294 if isdeb; then
 295     apt-get -y install php-wikidiff2
 296     teeu $mwc <<'EOF'
 297 $wgExternalDiffEngine = 'wikidiff2';
 298 EOF
 299     dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d
 300     ln -sf ../../mods-available/wikidiff2.ini $dir
 301     service apache2 restart
 302 fi
 303 # </source>
 304 # <source lang="bash">
 305 mw-ext Math
 306 # php5-curl according to Math readme
 307 if isdeb; then
 308     curl_pkg=php7.0-curl
 309     if ! apt-get -s install $curl_pkg &>/dev/null; then
 310         curl_pkg=php5-curl
 311     fi
 312     apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base \
 313             ghostscript imagemagick ocaml $curl_pkg make
 314 else
 315     # todo, php5-curl equivalent on fedora
 316     yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml
 317 fi
 318 service apache2 restart
 319
 320 cd $mw/extensions/Math/math; make # makes texvc
 321 cd $mw/extensions/Math/texvccheck; make
 322
 323 teeu $mwc <<'EOF'
 324 # Enable MathJax as rendering option
 325 $wgUseMathJax = true;
 326 # Enable LaTeXML as rendering option
 327 $wgMathValidModes[] = 'latexml';
 328 # Set LaTeXML as default rendering option, because it is nicest
 329 $wgDefaultUserOptions['math'] = 'latexml';
 330 EOF
 331 # </source>
 332 # <source lang="bash">
 333 mw-ext SpamBlacklist
 334 if ! grep -F '$wgSpamBlacklistFiles = array(' $mwc &>/dev/null; then
 335     tee -a $mwc <<'EOF'
 336 $wgEnableDnsBlacklist = true;
 337 $wgDnsBlacklistUrls = array( 'xbl.spamhaus.org', 'dnsbl.tornevall.org' );
 338
 339 ini_set( 'pcre.backtrack_limit', '10M' );
 340 $wgSpamBlacklistFiles = array(
 341    "[[m:Spam blacklist]]",
 342    "http://en.wikipedia.org/wiki/MediaWiki:Spam-blacklist"
 343 );
 344 EOF
 345 fi
 346 # </source>
 347 # <source lang="bash">
 348 mw-ext TitleBlacklist
 349 if ! grep -F '$wgTitleBlacklistSources = array(' $mwc &>/dev/null; then
 350     tee -a $mwc <<'EOF'
 351 $wgTitleBlacklistSources = array(
 352     array(
 353          'type' => 'local',
 354          'src'  => 'MediaWiki:Titleblacklist',
 355     ),
 356     array(
 357          'type' => 'url',
 358          'src'  => 'http://meta.wikimedia.org/w/index.php?title=Title_blacklist&action=raw',
 359     ),
 360 );
 361 EOF
 362 fi
 363 # </source>
 364 # <source lang="bash">
 365 mw-ext WikiEditor
 366 teeu $mwc <<'EOF'
 367 # Enable Wikieditor by default
 368 $wgDefaultUserOptions['usebetatoolbar'] = 1;
 369 $wgDefaultUserOptions['usebetatoolbar-cgd'] = 1;
 370
 371 # Display the Preview and Changes tabs
 372 $wgDefaultUserOptions['wikieditor-preview'] = 1;
 373 EOF
 374 # </source>
 375 # <source lang="bash">
 376 mw-ext CategoryTree
 377 teeu $mwc <<'EOF'
 378 # Mediawiki setting dependency for CategoryTree
 379 $wgUseAjax = true;
 380 EOF
 381 # </source>
 382 # <source lang="bash">
 383 mw-ext AbuseFilter
 384 teeu $mwc<<'EOF'
 385 $wgGroupPermissions['sysop']['abusefilter-modify'] = true;
 386 $wgGroupPermissions['*']['abusefilter-log-detail'] = true;
 387 $wgGroupPermissions['*']['abusefilter-view'] = true;
 388 $wgGroupPermissions['*']['abusefilter-log'] = true;
 389 $wgGroupPermissions['sysop']['abusefilter-private'] = true;
 390 $wgGroupPermissions['sysop']['abusefilter-modify-restricted'] = true;
 391 $wgGroupPermissions['sysop']['abusefilter-revert'] = true;
 392 EOF
 393 # </source>
 394 # <source lang="bash">
 395 mw-ext ConfirmEdit
 396 captchaArray
 397 teeu $mwc <<'EOF'
 398 wfLoadExtension( 'ConfirmEdit/QuestyCaptcha' );
 399 $wgCaptchaClass = 'QuestyCaptcha';
 400 # only captcha on registration
 401 $wgGroupPermissions['user'         ]['skipcaptcha'] = true;
 402 $wgGroupPermissions['autoconfirmed']['skipcaptcha'] = true;
 403 EOF
 404 if ! grep -Fx 'foreach ( $localSettingsQuestyQuestions as $key => $value ) {' $mwc; then
 405     tee -a $mwc <<'EOF'
 406 foreach ( $localSettingsQuestyQuestions as $key => $value ) {
 407         $wgCaptchaQuestions[] = array( 'question' => $key, 'answer' => $value );
 408 }
 409 EOF
 410 fi
 411 # </source>
 412 # <source lang="bash">
 413 sed -i --follow-symlinks "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
 414 # </source>
 415 # <source lang="bash">
 416 # get repo
 417 if [[ ! -e ~/pywikibot/.git ]]; then
 418     git clone --recursive \
 419         https://gerrit.wikimedia.org/r/pywikibot/core.git ~/pywikibot
 420 fi
 421 cd ~/pywikibot
 422 #updating
 423 git pull --all
 424 git submodule update
 425 # </source>
 426 # <source lang="bash">
 427 cd $HOME/pywikibot
 428 dd of=user-config.py <<EOF
 429 mylang = 'en'
 430 usernames["$mwfamily"]['en'] = u'$wikiuser'
 431 family = "$mwfamily"
 432 console_encoding = 'utf-8'
 433 password_file = "secretsfile"
 434 EOF
 435
 436 dd of=secretsfile <<EOF
 437 ("$wikiuser", "$wikipass")
 438 EOF
 439
 440 # it won't overrwrite an existing file. Remove if if one exists
 441 rm -f  pywikibot/families/${mwfamily}_family.py
 442 if isdeb; then
 443     apt-get install -y python-requests
 444 else
 445     yum -y install python-requests
 446 fi
 447
 448 python generate_family_file.py https://$mwdomain/wiki/Main_Page "$mwfamily"
 449
 450 # Note, this needed only for ssl site
 451 tee -a pywikibot/families/${mwfamily}_family.py<<'EOF'
 452     def protocol(self, code):
 453         return 'https'
 454 EOF
 455 # </source>
 456 # <source lang="bash">
 457 cd "$HOME/pywikibot"
 458
 459 dd of=scripts/${mwfamily}_setup.py<<EOF
 460 import pywikibot
 461 import time
 462 import sys
 463 site = pywikibot.Site()
 464 def x(p, t=""):
 465    page = pywikibot.Page(site, p)
 466    page.text = t
 467    #force is for some anti-bot thing, not necessary in my testing, but might as well include it
 468    page.save(force=True)
 469
 470 # Small/medium noncommercial wiki should be fine with no privacy policy
 471 # based on https://www.mediawiki.org/wiki/Manual:Footer
 472 x("MediaWiki:Privacy")
 473
 474 # licenses for uploads. Modified from the mediawiki's wiki
 475 x("MediaWiki:Licenses", u"""* Same as this wiki's text (preferred)
 476 ** CC BY-SA or GFDL| Creative Commons Attribution ShareAlike or GNU Free Documentation License
 477 * Others:
 478 ** Unknown_copyright|I don't know exactly
 479 ** PD|PD: public domain
 480 ** CC BY|Creative Commons Attribution
 481 ** CC BY-SA|Creative Commons Attribution ShareAlike
 482 ** GFDL|GFDL: GNU Free Documentation License
 483 ** GPL|GPL: GNU General Public License
 484 ** LGPL|LGPL: GNU Lesser General Public License""")
 485 x("MediaWiki:Copyright", '$mw_license')
 486 x("MediaWiki:Mainpage-description", "$mwdescription")
 487
 488
 489
 490 # The rest of the settings are for the site style
 491
 492 # Remove various clutter
 493 x("MediaWiki:Lastmodifiedat")
 494 x("MediaWiki:Disclaimers")
 495 x("MediaWiki:Viewcount")
 496 x("MediaWiki:Aboutsite")
 497 # remove these lines from sidebar
 498 # ** recentchanges-url|recentchanges
 499 # ** randompage-url|randompage
 500 # ** helppage|help
 501 x("MediaWiki:Sidebar", """* navigation
 502 ** mainpage|mainpage-description
 503 * SEARCH
 504 * TOOLBOX
 505 * LANGUAGES""")
 506
 507 # remove side panel
 508 # helpfull doc: https://www.mediawiki.org/wiki/Manual:Interface/Sidebar
 509 x("mediawiki:Common.css", """/* adjust sidebar to just be home link and up top  */
 510 /* panel width increased to fit full wiki name. */
 511 div#mw-panel { top: 10px; padding-top: 0em; width: 20em }
 512 div#footer, #mw-head-base, div#content { margin-left: 1em; }
 513 #left-navigation { margin-left: 1em; }
 514
 515
 516 /* logo, and toolbar hidden */
 517 #p-logo, #p-tb.portal {
 518    display:none;
 519 }
 520
 521 /* make the font size smaller for the misc stuff */
 522 #p-personal {
 523    font-size: 0.8em;
 524 }
 525
 526 #footer-info {
 527    font-size: 0.8em;
 528 }
 529 div#mw-content-text {
 530     max-width: 720px;
 531 }
 532 """)
 533 EOF
 534
 535 # this can spam a warning, so uniq it
 536 python pwb.py ${mwfamily}_setup |& uniq
 537 # </source>
 538 # <source lang="bash">
 539 s=/etc/cron.daily/mediawiki_update
 540 dd of=$s<<'EOF'
 541 #!/bin/bash
 542 source ~/mw_vars
 543 cd $mw
 544 for dir in extensions/* skins/* $mw; do
 545     [[ -d $dir ]] || continue
 546     cd $dir
 547     branch=$(git describe --all)
 548     branch=${branch#remotes/}
 549     git fetch --all -q
 550     new_head=$(git rev-parse $branch)
 551     log=$(git log HEAD..$new_head)
 552     if [[ $log ]]; then
 553         pwd
 554         echo "$log"
 555     fi
 556     git checkout -q $new_head
 557     cd $mw
 558 done
 559 php $mw/maintenance/update.php -q
 560 EOF
 561
 562 # </source>