iankelling.org Git - mediawiki-setup/blob - mw-setup-script

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3 # This program is under GPL v. 3 or later, see <http://www.gnu.org/licenses/>
   4 set -x
   5 # <source lang="bash">
   6 # identify if this is a debian based distro
   7 isdeb() { command -v apt &>/dev/null; }
   8 # tee unique. append each stdin line if it does not exist in the file
   9 teeu () {
  10     local MAPFILE
  11     mapfile -t
  12     for line in "${MAPFILE[@]}"; do
  13         grep -xFq "$line" "$1" &>/dev/null || tee -a "$1" <<<"$line"
  14     done
  15 }
  16
  17 # get and reset an extension/skin repository, and enable it
  18 mw-clone() {
  19     local url=$1
  20     local original_pwd="$PWD"
  21     local name
  22     local re='[^/]*/[^/]*$'
  23     [[ $url =~ $re ]] ||:
  24     target=$mw/${BASH_REMATCH[0]}
  25     if [[ ! -e $target/.git ]]; then
  26         git clone $url $target
  27     fi
  28     if ! cd $target; then
  29         echo "mw-ext error: failed cd $target";
  30         exit 1
  31     fi
  32     git fetch
  33     git checkout -qf origin/$mw_branch || git checkout -qf origin/master
  34     git clean -xffd
  35     cd "$original_pwd"
  36
  37 }
  38 mw-ext () {
  39     local ext
  40     for ext in "$@"; do
  41         mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/$ext
  42         if [[ -e $mw/ext/$ext/extension.json ]]; then
  43             # new style extension. remove old style declaration
  44             sed -i --follow-symlinks '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc
  45             teeu $mwc <<EOF
  46 wfLoadExtension( '$ext' );
  47 EOF
  48         else
  49             teeu $mwc <<EOF
  50 require_once( "\$IP/extensions/$ext/$ext.php" );
  51 EOF
  52         fi
  53     done
  54     # --quick is quicker than default flags,
  55     # but still add a sleep to make sure everything works right
  56     sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
  57 }
  58 mw-skin() {
  59     local skin=$1
  60     mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/skins/$skin
  61     sed -i --follow-symlinks '/^wfLoadSkin/d' $mwc
  62     sed -i --follow-symlinks '/^\$wgDefaultSkin/d' $mwc
  63     teeu $mwc <<EOF
  64 \$wgDefaultSkin = "${skin,,*}";
  65 wfLoadSkin( '$skin' );
  66 EOF
  67     sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
  68 }
  69
  70 if command -v apt &>/dev/null; then
  71     apache_user=www-data
  72 else
  73     apache_user=apache
  74 fi
  75
  76 # </source>
  77 # <source lang="bash">
  78 # From here on out, exit if a command fails.
  79 # This will prevent us from not noticing an important failure.
  80 # We recommend setting this for the entire installation session.
  81 # If you are running commands interactively, it might be best to
  82 # put it in your ~/.bashrc temporarily.
  83 set -eE -o pipefail
  84 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
  85 source ~/mw_vars
  86
  87 if isdeb; then
  88     # main reference:
  89     # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu
  90     apt-get update
  91     DEBIAN_FRONTEND=noninteractive apt-get install -y imagemagick curl
  92     if apt-get install -s mediawiki &>/dev/null; then
  93         # mediawiki is packaged in jessie backports.
  94         DEBIAN_FRONTEND=noninteractive apt-get -y install php5-apcu mediawiki
  95     else
  96         # https://www.mediawiki.org/wiki/Manual:Installation_requirements
  97         if apt-get install -s php7.0 &>/dev/null; then
  98             # note, 7.0 is untested by the editor here, since it's not
  99             # available in debian 8. it's listed as supported
 100             # in the mediawiki page.
 101             # noninteractive to avoid mysql password prompt.
 102             DEBIAN_FRONTEND=noninteractive apt-get install -y apache2  \
 103                            default-mysql-server \
 104                            php7.0 php7.0-mysql libapache2-mod-php7.0 php7.0-xml \
 105                            php7.0-apcu php7.0-mbstring
 106         else
 107             # note: mbstring is recommended, but it's not available for php5 in
 108             # debian jessie.
 109             DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 \
 110                            default-mysql-server \
 111                            php5 php5-mysql libapache2-mod-php5 php5-apcu
 112         fi
 113     fi
 114     service apache2 restart
 115 else
 116     # note
 117     # fedora deps are missing a database, so some is translated from debian packages
 118     yum -y install mediawiki ImageMagick php-mysqlnd php-pecl-apcu mariadb-server
 119
 120     systemctl restart mariadb.service
 121     systemctl enable mariadb.service
 122     systemctl enable httpd.service
 123     systemctl restart httpd.service
 124 fi
 125
 126
 127 # slightly different depending on if we already set the root pass
 128 if echo exit|mysql -u root -p"$dbpass"; then
 129     # answer interactive prompts:
 130     # mysql root pass, change pass? no, remove anon users? (default, yes)
 131     # disallow remote root (default, yes), reload? (default, yes)
 132     echo -e "$dbpass\nn\n\n\n\n" | mysql_secure_installation
 133 else
 134     # I had 1 less newline at the start when doing ubuntu 14.04,
 135     # compared to debian 8, so can't say this is especially portable.
 136     # It won't hurt if it fails.
 137     echo -e "\n\n$dbpass\n$dbpass\n\n\n\n\n" | mysql_secure_installation
 138 fi
 139 # </source>
 140 # <source lang="bash">
 141 mkdir -p $mw
 142 cd $mw
 143 # this will just fail if it already exists which is fine
 144 if [[ ! -e .git ]]; then
 145     git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
 146 fi
 147 # to see available branches: https://www.mediawiki.org/wiki/Version_lifecycle
 148 # and
 149 # git branch -r
 150 git checkout -f origin/$mw_branch
 151 git clean -ffxd
 152 # add some librejs support:
 153 curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-librejs-patch;hb=HEAD" | bash
 154 # Get the php libraries wmf uses. Based on:
 155 # https://www.mediawiki.org/wiki/Download_from_Git#Fetch_external_libraries
 156 if [[ ! -e vendor/.git ]]; then
 157     git clone  https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
 158 fi
 159 cd vendor
 160 git checkout -f origin/$mw_branch
 161 cd ..
 162
 163 # Drop any previous database which may have been installed while testing.
 164 # If upgrading, we should have a db backup which will get restored.
 165 # https://www.mediawiki.org/wiki/Manual:Upgrading
 166 mysql -u root -p$dbpass <<'EOF' ||:
 167 drop database my_wiki;
 168 exit
 169 EOF
 170 php $mw/maintenance/install.php --pass $wikipass --scriptpath /w \
 171     --dbuser root --dbpass $dbpass "$mwdescription" "$wikiuser"
 172 teeu $mwc <<'EOF'
 173 # lock down the wiki to only the initial owner until anti-spam measures are put in place
 174 # limit edits to registered users
 175 $wgGroupPermissions['*']['edit'] = false;
 176 # don't allow any account creation
 177 $wgGroupPermissions['*']['createaccount'] = false;
 178 EOF
 179 # </source>
 180 # <source lang="bash">
 181 temp=$(mktemp -d)
 182 cd $temp
 183 git_site=https://iankelling.org/git
 184 git clone $git_site/acme-tiny-wrapper
 185 l=$mw/../../logs
 186 mkdir -p $l
 187
 188 acme-tiny-wrapper/acme-tiny-wrapper -t $mwdomain
 189
 190 git clone $git_site/basic-https-conf
 191 { cat <<EOF
 192 ServerAdmin $mw_email
 193 RewriteEngine On
 194 # make the site's root url go to our main page
 195 RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
 196 # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
 197 RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
 198 EOF
 199 find -L $(readlink -f $mw) -name .htaccess \
 200     | while read line; do
 201     echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
 202 done
 203 } | basic-https-conf/apache-site -r ${mw%/*} - $mwdomain
 204 cd
 205 rm -rf $temp
 206 # </source>
 207 # <source lang="bash">
 208 dd of=$mw/../robots.txt <<'EOF'
 209         User-agent: *
 210 Disallow: /w/
 211         User-agent: ia_archiver
 212 Allow: /*&action=raw
 213 EOF
 214 mw-skin Vector
 215 # </source>
 216 # <source lang="bash">
 217 teeu $mwc<<EOF
 218 \$wgServer = "https://$mwdomain";
 219 \$wgDBserver = "localhost";
 220 \$wgRightsUrl = "$mw_RightsUrl";
 221 \$wgRightsText = "$mw_RightsText";
 222 \$wgRightsIcon = "$mw_RightsIcon";
 223 EOF
 224 # </source>
 225 # <source lang="bash">
 226 teeu $mwc<<EOF
 227 \$wgPasswordSender = "$mw_email";
 228 \$wgEmergencyContact = "$mw_email";
 229 \$wgEnotifUserTalk = true; # UPO
 230 \$wgEnotifWatchlist = true; # UPO
 231 \$wgMainCacheType = CACHE_ACCEL;
 232 \$wgEnableUploads = true;
 233 \$wgUseInstantCommons = true;
 234 \$wgPingback = true;
 235 EOF
 236 # </source>
 237 # <source lang="bash">
 238 teeu $mwc <<'EOF'
 239 # from https://www.mediawiki.org/wiki/Manual:Short_URL
 240 $wgArticlePath = "/wiki/$1";
 241
 242 # https://www.mediawiki.org/wiki/Manual:Combating_spam
 243 # check that url if our precautions don't work
 244 # not using nofollow is good practice, as long as we avoid spam.
 245 $wgNoFollowLinks = false;
 246 # Allow user customization.
 247 $wgAllowUserCss = true;
 248 # use imagemagick over GD
 249 $wgUseImageMagick = true;
 250 # manual says this is not production ready, I think that is mostly
 251 # because they are using MobileFrontend extension instead, which gives
 252 # an even cleaner more minimal view, I plan to try setting it up
 253 # sometime but this seems like a very nice improvement for now.
 254 $wgVectorResponsive = true;
 255 EOF
 256
 257
 258 # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads
 259 # Increase from default of 2M to 100M.
 260 # This will at least allow high res pics etc.
 261 php_ini=$(php -r 'echo(php_ini_loaded_file());')
 262 sed -i --follow-symlinks 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini
 263 if isdeb; then
 264     service apache2 restart
 265 else
 266     systemctl restart httpd.service
 267 fi
 268
 269 # if you were to install as a normal user, you would need this for images
 270 # sudo usermod -aG $apache_user $USER
 271
 272 # this doesn't propogate right away
 273 chgrp -R $apache_user $mw/images
 274 chmod -R g+w $mw/images
 275 # </source>
 276 # <source lang="bash">
 277 teeu $mwc <<'EOF'
 278 $wgLogo = null;
 279 #$wgFooterIcons = null;
 280 EOF
 281 # Make the toolbox go into the drop down.
 282 cd $mw/skins/Vector
 283 if ! git remote show ian-kelling &>/dev/null; then
 284     git remote add ian-kelling https://iankelling.org/git/forks/Vector
 285 fi
 286 git fetch ian-kelling
 287 git checkout ian-kelling/${mw_branch}-toolbox-in-dropdown
 288 # </source>
 289 # <source lang="bash">
 290 mw-ext Cite CiteThisPage CheckUser CSS Echo Gadgets ImageMap Interwiki News \
 291        Nuke ParserFunctions Poem Renameuser SyntaxHighlight_GeSHi Variables
 292 # </source>
 293 # <source lang="bash">
 294 mw-ext AntiSpoof
 295 # recommended setup script to account for existing users
 296 sudo -u $apache_user php $mw/extensions/AntiSpoof/maintenance/batchAntiSpoof.php
 297 # </source>
 298 # <source lang="bash">
 299 if isdeb; then
 300     apt-get -y install php-wikidiff2
 301     teeu $mwc <<'EOF'
 302 $wgExternalDiffEngine = 'wikidiff2';
 303 EOF
 304     dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d
 305     ln -sf ../../mods-available/wikidiff2.ini $dir
 306     service apache2 restart
 307 fi
 308 # </source>
 309 # <source lang="bash">
 310 mw-ext Math
 311 # php5-curl according to Math readme
 312 if isdeb; then
 313     curl_pkg=php7.0-curl
 314     if ! apt-get -s install $curl_pkg &>/dev/null; then
 315         curl_pkg=php5-curl
 316     fi
 317     apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base \
 318             ghostscript imagemagick ocaml $curl_pkg make
 319 else
 320     # todo, php5-curl equivalent on fedora
 321     yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml
 322 fi
 323 service apache2 restart
 324
 325 cd $mw/extensions/Math/math; make # makes texvc
 326 cd $mw/extensions/Math/texvccheck; make
 327
 328 teeu $mwc <<'EOF'
 329 # Enable MathJax as rendering option
 330 $wgUseMathJax = true;
 331 # Enable LaTeXML as rendering option
 332 $wgMathValidModes[] = 'latexml';
 333 # Set LaTeXML as default rendering option, because it is nicest
 334 $wgDefaultUserOptions['math'] = 'latexml';
 335 EOF
 336 # </source>
 337 # <source lang="bash">
 338 mw-ext SpamBlacklist
 339 if ! grep -F '$wgSpamBlacklistFiles = array(' $mwc &>/dev/null; then
 340     tee -a $mwc <<'EOF'
 341 $wgEnableDnsBlacklist = true;
 342 $wgDnsBlacklistUrls = array( 'xbl.spamhaus.org', 'dnsbl.tornevall.org' );
 343
 344 ini_set( 'pcre.backtrack_limit', '10M' );
 345 $wgSpamBlacklistFiles = array(
 346    "[[m:Spam blacklist]]",
 347    "http://en.wikipedia.org/wiki/MediaWiki:Spam-blacklist"
 348 );
 349 EOF
 350 fi
 351 # </source>
 352 # <source lang="bash">
 353 mw-ext TitleBlacklist
 354 if ! grep -F '$wgTitleBlacklistSources = array(' $mwc &>/dev/null; then
 355     tee -a $mwc <<'EOF'
 356 $wgTitleBlacklistSources = array(
 357     array(
 358          'type' => 'local',
 359          'src'  => 'MediaWiki:Titleblacklist',
 360     ),
 361     array(
 362          'type' => 'url',
 363          'src'  => 'http://meta.wikimedia.org/w/index.php?title=Title_blacklist&action=raw',
 364     ),
 365 );
 366 EOF
 367 fi
 368 # </source>
 369 # <source lang="bash">
 370 mw-ext WikiEditor
 371 teeu $mwc <<'EOF'
 372 # Enable Wikieditor by default
 373 $wgDefaultUserOptions['usebetatoolbar'] = 1;
 374 $wgDefaultUserOptions['usebetatoolbar-cgd'] = 1;
 375
 376 # Display the Preview and Changes tabs
 377 $wgDefaultUserOptions['wikieditor-preview'] = 1;
 378 EOF
 379 # </source>
 380 # <source lang="bash">
 381 mw-ext CategoryTree
 382 teeu $mwc <<'EOF'
 383 # Mediawiki setting dependency for CategoryTree
 384 $wgUseAjax = true;
 385 EOF
 386 # </source>
 387 # <source lang="bash">
 388 mw-ext AbuseFilter
 389 teeu $mwc<<'EOF'
 390 $wgGroupPermissions['sysop']['abusefilter-modify'] = true;
 391 $wgGroupPermissions['*']['abusefilter-log-detail'] = true;
 392 $wgGroupPermissions['*']['abusefilter-view'] = true;
 393 $wgGroupPermissions['*']['abusefilter-log'] = true;
 394 $wgGroupPermissions['sysop']['abusefilter-private'] = true;
 395 $wgGroupPermissions['sysop']['abusefilter-modify-restricted'] = true;
 396 $wgGroupPermissions['sysop']['abusefilter-revert'] = true;
 397 EOF
 398 # </source>
 399 # <source lang="bash">
 400 mw-ext ConfirmEdit
 401 captchaArray
 402 teeu $mwc <<'EOF'
 403 wfLoadExtension( 'ConfirmEdit/QuestyCaptcha' );
 404 $wgCaptchaClass = 'QuestyCaptcha';
 405 # only captcha on registration
 406 $wgGroupPermissions['user'         ]['skipcaptcha'] = true;
 407 $wgGroupPermissions['autoconfirmed']['skipcaptcha'] = true;
 408 EOF
 409 if ! grep -Fx 'foreach ( $localSettingsQuestyQuestions as $key => $value ) {' $mwc; then
 410     tee -a $mwc <<'EOF'
 411 foreach ( $localSettingsQuestyQuestions as $key => $value ) {
 412         $wgCaptchaQuestions[] = array( 'question' => $key, 'answer' => $value );
 413 }
 414 EOF
 415 fi
 416 # </source>
 417 # <source lang="bash">
 418 sed -i --follow-symlinks "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
 419 # </source>
 420 # <source lang="bash">
 421 # get repo
 422 if [[ ! -e ~/pywikibot/.git ]]; then
 423     git clone --recursive \
 424         https://gerrit.wikimedia.org/r/pywikibot/core.git ~/pywikibot
 425 fi
 426 cd ~/pywikibot
 427 #updating
 428 git pull --all
 429 git submodule update
 430 # </source>
 431 # <source lang="bash">
 432 cd $HOME/pywikibot
 433 dd of=user-config.py <<EOF
 434 mylang = 'en'
 435 usernames["$mwfamily"]['en'] = u'$wikiuser'
 436 family = "$mwfamily"
 437 console_encoding = 'utf-8'
 438 password_file = "secretsfile"
 439 EOF
 440
 441 dd of=secretsfile <<EOF
 442 ("$wikiuser", "$wikipass")
 443 EOF
 444
 445 # it won't overrwrite an existing file. Remove if if one exists
 446 rm -f  pywikibot/families/${mwfamily}_family.py
 447 if isdeb; then
 448     apt-get install -y python-requests
 449 else
 450     yum -y install python-requests
 451 fi
 452
 453 python generate_family_file.py https://$mwdomain/wiki/Main_Page "$mwfamily"
 454
 455 # Note, this needed only for ssl site
 456 tee -a pywikibot/families/${mwfamily}_family.py<<'EOF'
 457     def protocol(self, code):
 458         return 'https'
 459 EOF
 460 # </source>
 461 # <source lang="bash">
 462 cd "$HOME/pywikibot"
 463
 464 dd of=scripts/${mwfamily}_setup.py<<EOF
 465 import pywikibot
 466 import time
 467 import sys
 468 site = pywikibot.Site()
 469 def x(p, t=""):
 470    page = pywikibot.Page(site, p)
 471    page.text = t
 472    #force is for some anti-bot thing, not necessary in my testing, but might as well include it
 473    page.save(force=True)
 474
 475 # Small/medium noncommercial wiki should be fine with no privacy policy
 476 # based on https://www.mediawiki.org/wiki/Manual:Footer
 477 x("MediaWiki:Privacy")
 478
 479 # licenses for uploads. Modified from the mediawiki's wiki
 480 x("MediaWiki:Licenses", u"""* Same as this wiki's text (preferred)
 481 ** CC BY-SA or GFDL| Creative Commons Attribution ShareAlike or GNU Free Documentation License
 482 * Others:
 483 ** Unknown_copyright|I don't know exactly
 484 ** PD|PD: public domain
 485 ** CC BY|Creative Commons Attribution
 486 ** CC BY-SA|Creative Commons Attribution ShareAlike
 487 ** GFDL|GFDL: GNU Free Documentation License
 488 ** GPL|GPL: GNU General Public License
 489 ** LGPL|LGPL: GNU Lesser General Public License""")
 490 x("MediaWiki:Copyright", '$mw_license')
 491 x("MediaWiki:Mainpage-description", "$mwdescription")
 492
 493
 494
 495 # The rest of the settings are for the site style
 496
 497 # Remove various clutter
 498 x("MediaWiki:Lastmodifiedat")
 499 x("MediaWiki:Disclaimers")
 500 x("MediaWiki:Viewcount")
 501 x("MediaWiki:Aboutsite")
 502 # remove these lines from sidebar
 503 # ** recentchanges-url|recentchanges
 504 # ** randompage-url|randompage
 505 # ** helppage|help
 506 x("MediaWiki:Sidebar", """* navigation
 507 ** mainpage|mainpage-description
 508 * SEARCH
 509 * TOOLBOX
 510 * LANGUAGES""")
 511
 512 # remove side panel
 513 # helpfull doc: https://www.mediawiki.org/wiki/Manual:Interface/Sidebar
 514 x("mediawiki:Common.css", """/* adjust sidebar to just be home link and up top  */
 515 /* panel width increased to fit full wiki name. */
 516 div#mw-panel { top: 10px; padding-top: 0em; width: 20em }
 517 div#footer, #mw-head-base, div#content { margin-left: 1em; }
 518 #left-navigation { margin-left: 1em; }
 519
 520
 521 /* logo, and toolbar hidden */
 522 #p-logo, #p-tb.portal {
 523    display:none;
 524 }
 525
 526 /* make the font size smaller for the misc stuff */
 527 #p-personal {
 528    font-size: 0.8em;
 529 }
 530
 531 #footer-info {
 532    font-size: 0.8em;
 533 }
 534 div#mw-content-text {
 535     max-width: 720px;
 536 }
 537 """)
 538 EOF
 539
 540 # this can spam a warning, so uniq it
 541 python pwb.py ${mwfamily}_setup |& uniq
 542 # </source>
 543 # <source lang="bash">
 544 s=/etc/cron.daily/mediawiki_update
 545 dd of=$s<<'EOF'
 546 #!/bin/bash
 547 source ~/mw_vars
 548 update() {
 549     dir=$1
 550     cd $mw
 551     [[ -d $dir ]] || return 1
 552     cd $dir
 553     branch=$(git describe --all)
 554     branch=${branch#remotes/}
 555     git fetch --all -q
 556     new_head=$(git rev-parse $branch)
 557     log=$(git log HEAD..$new_head)
 558     if [[ ! $log ]]; then
 559         return 1
 560     fi
 561     pwd
 562     echo "$log"
 563     git checkout -qf $new_head
 564     cd $mw
 565     return 0
 566 }
 567 for dir in extensions/* skins/* vendor; do
 568     update "$dir" ||:
 569 done
 570 if update .; then
 571     curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-librejs-patch;hb=HEAD" | bash
 572 fi
 573 php $mw/maintenance/update.php -q --quick
 574 EOF
 575
 576 # </source>