151d9495643ede6662fa481adbf91a318abb101f
[mediawiki-setup] / mw-setup-script
1 #!/bin/bash
2 # Copyright (C) 2016 Ian Kelling
3 # This program is under GPL v. 3 or later, see <http://www.gnu.org/licenses/>
4 # <source lang="bash">
5 # for convenience, Mediawiki config file
6 mwc="$mw/LocalSettings.php"
7
8 if isdeb; then
9 apache_user=www-data
10 else
11 apache_user=apache
12 fi
13
14 # get and reset an extension/skin repository, and enable it
15 mw-ext () { mw-extra extension $@; }
16 mw-skin() { mw-extra skin $@; }
17
18 mw-extra() {
19 local type=${1}s # extension or skin
20 shift
21 local clone=true
22 if [[ $1 == -g ]]; then
23 clone=false
24 shift
25 fi
26 local ext
27 for ext in "$@"; do
28 if $clone; then
29 local original_pwd="$PWD"
30 # it's ok that this fails if we already have it
31 url=https://git.wikimedia.org/git/mediawiki
32 target=$mw/$type/$ext
33 git clone $url/$type/$ext.git $target
34 if ! cd $target; then
35 echo "mw-ext error: failed cd $mw/extensions/$ext";
36 exit 1
37 fi
38 git fetch
39 git checkout -qf origin/$mw_branch || git checkout -qf origin/master
40 git clean -xffd
41 cd "$original_pwd"
42 fi
43 case $type in
44 extensions)
45 if [[ -e $target/extension.json ]]; then
46 # new style extension. remove old style declaration
47 sed -i '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc
48 teeu $mwc <<EOF
49 wfLoadExtension( '$ext' );
50 EOF
51 else
52 teeu $mwc <<EOF
53 require_once( "\$IP/extensions/$ext/$ext.php" );
54 EOF
55 fi
56 ;;
57 skins)
58 sed -i '/^wfLoadSkin/d' $mwc
59 sed -i '/^\$wgDefaultSkin/d' $mwc
60 teeu $mwc <<EOF
61 $wgDefaultSkin = "${ext,,*}";
62 wfLoadSkin( 'Vector' );
63 EOF
64 ;;
65 esac
66 done
67 # --quick is quicker than default flags,
68 # but still add a sleep to make sure everything works right
69 sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
70 }
71 # </source>
72 # <source lang="bash">
73 if isdeb; then
74 apt-get update
75 # noninteractive to avoid mysql password prompt
76 DEBIAN_FRONTEND=noninteractive apt-get -y install mediawiki ImageMagick php5-mysqlnd php-apc
77 service apache2 restart
78 else
79 # fedora deps are missing a database, so some is translated from debian packages
80 yum -y install mediawiki ImageMagick php-mysqlnd php-pecl-apcu mariadb-server
81
82 systemctl restart mariadb.service
83 systemctl enable mariadb.service
84 systemctl enable httpd.service
85 systemctl restart httpd.service
86 fi
87
88
89 # slightly different depending on if we already set the root pass
90 if echo exit|mysql -u root -p"$dbpass"; then
91 # answer interactive prompts:
92 # mysql root pass, change pass? no, remove anon users? (default, yes)
93 # disallow remote root (default, yes), reload? (default, yes)
94 echo -e "$dbpass\nn\n\n\n\n" | mysql_secure_installation
95 else
96 # I had 1 less newline at the start when doing ubuntu 14.04,
97 # compared to debian 8, so can't say this is especially portable.
98 # It won't hurt if it fails.
99 echo -e "\n\n$dbpass\n$dbpass\n\n\n\n\n" | mysql_secure_installation
100 fi
101 # </source>
102 # <source lang="bash">
103 mkdir -p $mw
104 cd $mw
105 # this will just fail if it already exists which is fine
106 git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
107 # to see available branches: https://www.mediawiki.org/wiki/Version_lifecycle
108 # and
109 # git branch -r
110 git checkout -f origin/$mw_branch
111 git clean -ffxd
112 # Get the php libraries wmf uses. Based on:
113 # https://www.mediawiki.org/wiki/Download_from_Git#Fetch_external_libraries
114 git clone https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
115 cd vendor
116 git checkout -f origin/$mw_branch
117 cd ..
118
119 # Drop any previous database which may have been installed while testing.
120 # If upgrading, we should have a db backup which will get restored.
121 # https://www.mediawiki.org/wiki/Manual:Upgrading
122 mysql -u root -p$dbpass <<'EOF'
123 drop database my_wiki;
124 exit
125 EOF
126 php $mw/maintenance/install.php --pass $wikipass --scriptpath /w \
127 --dbuser root --dbpass $dbpass "$mwdescription" "$wikiuser"
128 teeu $mwc <<'EOF'
129 # lock down the wiki to only the initial owner until anti-spam measures are put in place
130 # limit edits to registered users
131 $wgGroupPermissions['*']['edit'] = false;
132 # don't allow any account creation
133 $wgGroupPermissions['*']['createaccount'] = false;
134 EOF
135 # </source>
136 # <source lang="bash">
137 tmpdir="$(mktemp -d)"
138 cd $tmpdir
139 wget http://builds.piwik.org/piwik.zip
140 if isdeb; then apt-get -y install unzip; else yum -y install unzip; fi
141 unzip -q piwik.zip
142 # gui installer suggested command
143 if isdeb; then
144 chown -R www-data:www-data piwik
145 else
146 chown -R apache:apache piwik
147 fi
148 # remove any existing directory
149 rm -rf $mw/../analytics
150 mv piwik $mw/../analytics
151 rm -rf $tmpdir
152
153 # </source>
154 # <source lang="bash">
155 l=$mw/../../logs
156 mkdir $l
157
158 acme-tiny-wrapper $mwdomain
159 dd of=/etc/apache2/ports.conf <<'EOF'
160 Listen 8082
161 EOF
162 { cat <<EOF
163 ServerAdmin $mw_email
164 RewriteEngine On
165 # make the site's root url go to our main page
166 RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
167 # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
168 RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
169 EOF
170 find -L $(readlink -f $mw/..) -name .htaccess \
171 | while read line; do
172 echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
173 done
174 } | apache-site -i -p 127.0.0.1:8082 -- - $mwdomain
175 nginx-site -p 8082 $mwdomain
176 # </source>
177 # <source lang="bash">
178 if isdeb; then
179 a2ensite $mwdomain.conf
180 # for short urls
181 a2enmod rewrite
182 # We restart rather than reload because of the module
183 service apache2 restart
184 else
185 systemctl reload httpd.service
186 fi
187 # </source>
188 # <source lang="bash">
189 dd of=$mw/../robots.txt <<'EOF'
190 User-agent: *
191 Disallow: /w/
192 User-agent: ia_archiver
193 Allow: /*&action=raw
194 EOF
195 # </source>
196 # <source lang="bash">
197 teeu $mwc<<EOF
198 \$wgServer = "https://$mwdomain";
199 \$wgDBserver = "localhost";
200 \$wgRightsUrl = "$mw_RightsUrl";
201 \$wgRightsText = "$mw_RightsText";
202 \$wgRightsIcon = "$mw_RightsIcon";
203 wfLoadSkin( 'Vector' );
204 EOF
205 # </source>
206 # <source lang="bash">
207 teeu $mwc<<EOF
208 \$wgPasswordSender = "$mw_email";
209 \$wgEmergencyContact = "$mw_email";
210 \$wgEnotifUserTalk = true; # UPO
211 \$wgEnotifWatchlist = true; # UPO
212 \$wgMainCacheType = CACHE_ACCEL;
213 \$wgEnableUploads = true;
214 \$wgUseInstantCommons = true;
215 EOF
216 # </source>
217 # <source lang="bash">
218 teeu $mwc <<'EOF'
219 # from https://www.mediawiki.org/wiki/Manual:Short_URL
220 $wgArticlePath = "/wiki/$1";
221
222 # https://www.mediawiki.org/wiki/Manual:Combating_spam
223 # check that url if our precautions don't work
224 # not using nofollow is good practice, as long as we avoid spam.
225 $wgNoFollowLinks = false;
226 # Allow user customization.
227 $wgAllowUserJs = true;
228 $wgAllowUserCss = true;
229
230 # use imagemagick over GD
231 $wgUseImageMagick = true;
232 EOF
233
234
235 # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads
236 # Increase from default of 2M to 100M.
237 # This will at least allow high res pics etc.
238 php_ini=$(isdeb && echo /etc/php5/apache2/php.ini || echo /etc/php.ini)
239 sed -i 's/^\(upload_max_filesize\)\b.*/\1 = 100M/'
240 sed -i 's/^\(post_max_size\)\b.*/\1 = 100M/'
241 if isdeb; then
242 service apache2 restart
243 else
244 systemctl restart httpd.service
245 fi
246
247 # if you were to install as a normal user, you would need this for images
248 # sudo usermod -aG $apache_user $USER
249
250 # this doesn't propogate right away
251 chgrp -R $apache_user $mw/images
252 chmod -R g+w $mw/images
253 # </source>
254 # <source lang="bash">
255 teeu $mwc <<'EOF'
256 $wgLogo = null;
257 $wgShowIPinHeader = false;
258 $wgFooterIcons = null;
259 EOF
260 # Make the toolbox go into the drop down.
261 cd $mw
262 git remote add ian git@gitorious.org:mediawiki-toolbox-patch/mediawiki-toolbox-patch.git
263 git fetch ian
264 git rebase ian/REL1_23-toolbox-in-dropdown
265 # </source>
266 # <source lang="bash">
267 mw-ext Cite CiteThisPage CSS DynamicPageList Echo Gadgets ImageMap Interwiki News \
268 Nuke ParserFunctions Poem SyntaxHighlight_GeSHi Variables
269 # </source>
270 # <source lang="bash">
271 mw-ext AntiSpoof
272 # recommended setup script to account for existing users
273 sudo -u $apache_user php $mw/extensions/AntiSpoof/maintenance/batchAntiSpoof.php
274 # </source>
275 # <source lang="bash">
276 mw-ext CheckUser
277 sudo -u $apache_user php $mw/extensions/CheckUser/install.php; sleep 1
278 # </source>
279 # <source lang="bash">
280 if isdeb; then
281 apt-get -y install php-wikidiff2
282 teeu $mwc <<'EOF'
283 $wgExternalDiffEngine = 'wikidiff2';
284 EOF
285 ln -sf ../../mods-available/wikidiff2.ini /etc/php5/apache2/conf.d
286 service apache2 restart
287 fi
288 # </source>
289 # <source lang="bash">
290 mw-ext Math
291 # php5-curl according to Math readme
292 if isdeb; then
293 apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base ghostscript imagemagick ocaml php5-curl
294 else
295 # todo, php5-curl equivalent on fedora
296 yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml
297 fi
298 ln -sf ../../mods-available/curl.ini /etc/php5/apache2/conf.d
299 service apache2 restart
300
301 cd $mw/extensions/Math/math; make # makes texvc
302 cd $mw/extensions/Math/texvccheck; make
303
304 teeu $mwc <<'EOF'
305 # Enable MathJax as rendering option
306 $wgUseMathJax = true;
307 # Enable LaTeXML as rendering option
308 $wgMathValidModes[] = 'latexml';
309 # Set LaTeXML as default rendering option, because it is nicest
310 $wgDefaultUserOptions['math'] = 'latexml';
311 EOF
312 # </source>
313 # <source lang="bash">
314 mw-ext SpamBlacklist
315 if ! grep -F '$wgSpamBlacklistFiles = array(' $mwc &>/dev/null; then
316 tee -a $mwc <<'EOF'
317 $wgEnableDnsBlacklist = true;
318 $wgDnsBlacklistUrls = array( 'xbl.spamhaus.org', 'dnsbl.tornevall.org' );
319
320 ini_set( 'pcre.backtrack_limit', '10M' );
321 $wgSpamBlacklistFiles = array(
322 "[[m:Spam blacklist]]",
323 "http://en.wikipedia.org/wiki/MediaWiki:Spam-blacklist"
324 );
325 EOF
326 fi
327 # </source>
328 # <source lang="bash">
329 mw-ext TitleBlacklist
330 if ! grep -F '$wgTitleBlacklistSources = array(' $mwc &>/dev/null; then
331 tee -a $mwc <<'EOF'
332 $wgTitleBlacklistSources = array(
333 array(
334 'type' => 'local',
335 'src' => 'MediaWiki:Titleblacklist',
336 ),
337 array(
338 'type' => 'url',
339 'src' => 'http://meta.wikimedia.org/w/index.php?title=Title_blacklist&action=raw',
340 ),
341 );
342 EOF
343 fi
344 # </source>
345 # <source lang="bash">
346 mw-ext WikiEditor
347 teeu $mwc <<'EOF'
348 # Enable Wikieditor by default
349 $wgDefaultUserOptions['usebetatoolbar'] = 1;
350 $wgDefaultUserOptions['usebetatoolbar-cgd'] = 1;
351
352 # Display the Preview and Changes tabs
353 $wgDefaultUserOptions['wikieditor-preview'] = 1;
354 EOF
355 # </source>
356 # <source lang="bash">
357 mw-ext CategoryTree
358 teeu $mwc <<'EOF'
359 # Mediawiki setting dependency for CategoryTree
360 $wgUseAjax = true;
361 EOF
362 # </source>
363 # <source lang="bash">
364 mw-ext AbuseFilter
365 teeu $mwc<<'EOF'
366 $wgGroupPermissions['sysop']['abusefilter-modify'] = true;
367 $wgGroupPermissions['*']['abusefilter-log-detail'] = true;
368 $wgGroupPermissions['*']['abusefilter-view'] = true;
369 $wgGroupPermissions['*']['abusefilter-log'] = true;
370 $wgGroupPermissions['sysop']['abusefilter-private'] = true;
371 $wgGroupPermissions['sysop']['abusefilter-modify-restricted'] = true;
372 $wgGroupPermissions['sysop']['abusefilter-revert'] = true;
373 EOF
374 # </source>
375 # <source lang="bash">
376 mw-ext ConfirmEdit
377 captchaArray
378 teeu $mwc <<'EOF'
379 wfLoadExtension( 'ConfirmEdit/QuestyCaptcha' );
380 $wgCaptchaClass = 'QuestyCaptcha';
381 # only captcha on registration
382 $wgGroupPermissions['user' ]['skipcaptcha'] = true;
383 $wgGroupPermissions['autoconfirmed']['skipcaptcha'] = true;
384 EOF
385 if ! grep -Fx 'foreach ( $localSettingsQuestyQuestions as $key => $value ) {' $mwc; then
386 tee -a $mwc <<'EOF'
387 foreach ( $localSettingsQuestyQuestions as $key => $value ) {
388 $wgCaptchaQuestions[] = array( 'question' => $key, 'answer' => $value );
389 }
390 EOF
391 fi
392 # </source>
393 # <source lang="bash">
394 sed -i "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
395 # </source>
396 # <source lang="bash">
397 # get repo
398 mkdir ~/opt
399 git clone --recursive https://gerrit.wikimedia.org/r/pywikibot/core.git ~/opt/pywikibot
400 cd ~/opt/pywikibot
401 #updating
402 git pull --all
403 git submodule update
404 # </source>
405 # <source lang="bash">
406 cd $HOME/opt/pywikibot
407 dd of=user-config.py <<EOF
408 mylang = 'en'
409 usernames["$mwfamily"]['en'] = u'$wikiuser'
410 family = "$mwfamily"
411 console_encoding = 'utf-8'
412 password_file = "secretsfile"
413 EOF
414
415 dd of=secretsfile <<EOF
416 ("$wikiuser", "$wikipass")
417 EOF
418
419 # it won't overrwrite an existing file. Remove if if one exists
420 rm -f pywikibot/families/${mwfamily}_family.py
421 python generate_family_file.py https://$mwdomain/wiki/Main_Page "$mwfamily"
422
423 # Note, this needed only for ssl site
424 tee -a pywikibot/families/${mwfamily}_family.py<<'EOF'
425 def protocol(self, code):
426 return 'https'
427 EOF
428 # </source>
429 # <source lang="bash">
430 pw="$HOME/opt/pywikibot"
431
432 dd of=$pw/scripts/${mwfamily}_setup.py<<EOF
433 import pywikibot
434 import time
435 import sys
436 site = pywikibot.Site()
437 def x(p, t=""):
438 page = pywikibot.Page(site, p)
439 page.text = t
440 #force is for some anti-bot thing, not necessary in my testing, but might as well include it
441 page.save(force=True)
442
443 # Small/medium noncommercial wiki should be fine with no privacy policy
444 # based on https://www.mediawiki.org/wiki/Manual:Footer
445 x("MediaWiki:Privacy")
446
447 # licenses for uploads. Modified from the mediawiki's wiki
448 x("MediaWiki:Licenses", u"""* Same as this wiki's text (preferred)
449 ** CC BY-SA or GFDL| Creative Commons Attribution ShareAlike or GNU Free Documentation License
450 * Others:
451 ** Unknown_copyright|I don't know exactly
452 ** PD|PD: public domain
453 ** CC BY|Creative Commons Attribution
454 ** CC BY-SA|Creative Commons Attribution ShareAlike
455 ** GFDL|GFDL: GNU Free Documentation License
456 ** GPL|GPL: GNU General Public License
457 ** LGPL|LGPL: GNU Lesser General Public License""")
458 x("MediaWiki:Copyright", '$mw_license')
459 x("MediaWiki:Mainpage-description", "$mwdescription")
460
461
462
463 # The rest of the settings are for the site style
464
465 # Remove various clutter
466 x("MediaWiki:Lastmodifiedat")
467 x("MediaWiki:Disclaimers")
468 x("MediaWiki:Viewcount")
469 x("MediaWiki:Aboutsite")
470 # remove these lines from sidebar
471 # ** recentchanges-url|recentchanges
472 # ** randompage-url|randompage
473 # ** helppage|help
474 x("MediaWiki:Sidebar", """* navigation
475 ** mainpage|mainpage-description
476 * SEARCH
477 * TOOLBOX
478 * LANGUAGES""")
479
480 # remove side panel
481 # helpfull doc: https://www.mediawiki.org/wiki/Manual:Interface/Sidebar
482 x("mediawiki:Common.css", """/* adjust sidebar to just be home link and up top */
483 /* panel width increased to fit full wiki name. */
484 div#mw-panel { top: 10px; padding-top: 0em; width: 20em }
485 div#footer, #mw-head-base, div#content { margin-left: 1em; }
486 #left-navigation { margin-left: 1em; }
487
488
489 /* logo, and toolbar hidden */
490 #p-logo, #p-tb.portal {
491 display:none;
492 }
493
494 /* make the font size smaller for the misc stuff */
495 #p-personal {
496 font-size: 0.8em;
497 }
498
499 #footer-info {
500 font-size: 0.8em;
501 }
502 div#mw-content-text {
503 max-width: 720px;
504 }
505 """)
506 EOF
507
508 cd $pw
509 python pwb.py ${mwfamily}_setup
510 # </source>
511 # <source lang="bash">
512 #!/bin/bash
513 source ~/mw_vars
514
515 # if we get an error, keep going but return it at the end
516 last_error=0
517 trap 'last_error=$?' ERR
518
519 ssh root@$mwdomain <<ENDSSH
520 tee -a $mwc<<'EOF'
521 $wgReadOnly = 'Dumping Database, Access will be restored shortly';
522 EOF
523 mysqldump -p$dbpass --default-character-set=binary my_wiki > ~/wiki_backup/wikidump
524 sed -i '$ d' $mwc # delete last line
525 ENDSSH
526 rdiff-backup root@$mwdomain::/root/wiki_db_backup ~/backup/wiki_db_backup
527 rdiff-backup root@$mwdomain::$mw ~/backup/wiki_file_backup
528
529 exit $last_error
530 # </source>
531 # <source lang="bash">
532 chmod +x $HOME/bin/remote_wiki_backup
533 x="$(mktemp)"
534 crontab -l > "$x"
535 teeu "$x" <<'EOF'
536 0 4 * * * x=$($HOME/bin/remote_wiki_backup 2>&1); [[ $? != 0 ]] && echo "$x"
537 EOF
538 crontab "$x"
539 # </source>
540 # <source lang="bash">
541 source ~/mw_vars
542 HOSTNAME=REPLACE_ME source ~/mw_vars
543 rdiff-backup -r now ~/backup/wiki_file_backup /tmp/wiki_file_restore
544 scp -r /tmp/wiki_file_restore/images root@$mwdomain:$mw/images
545 rdiff-backup -r now ~/backup/wiki_db_backup /tmp/wiki_db_restore
546 scp -r /tmp/wiki_db_restore root@$mwdomain:/tmp
547 ssh root@$mwdomain "mysql -u root -p$dbpass my_wiki < /tmp/wiki_db_restore/wiki_db_dump"
548 # </source>
549 # <source lang="bash">
550 s=$HOME/bin/mediawiki_update
551 dd of=$s<<'EOF'
552 #!/bin/bash
553 source ~/mw_vars
554 cd $mw
555 git fetch --all
556 git checkout origin/$mw_branch
557 git rebase ian/REL1_23-toolbox-in-dropdown
558 cd extensions
559 for x in *; do
560 if [[ -d $x ]]; then
561 cd $x
562 git fetch --all
563 git checkout origin/$mw_branch || git checkout -qf origin/master
564 cd ..
565 fi
566 done
567 php $mw/maintenance/update.php -q
568 EOF
569 chmod +x $s
570
571 x="$(mktemp)"
572 crontab -l > "$x"
573 teeu "$x" <<EOF
574 0 5 * * * $s
575 EOF
576 crontab "$x"
577 # </source>
578 # <source lang="bash">
579 # docs suggests using separate database user, google lead me here for how
580 # https://www.digitalocean.com/community/tutorials/how-to-create-a-new-user-and-grant-permissions-in-mysql
581 mysql -u root -p$dbpass <<EOF
582 CREATE USER 'piwik'@'localhost' IDENTIFIED BY '$piwik_pass';
583 GRANT ALL PRIVILEGES ON piwik . * TO 'piwik'@'localhost';
584 FLUSH PRIVILEGES;
585 exit
586 EOF
587 # </source>
588 # <source lang="bash">
589 git clone https://github.com/DaSchTour/piwik-mediawiki-extension.git $mw/extensions/Piwik
590 mw-ext -g Piwik
591 teeu $mwc <<EOF
592 \$wgPiwikURL = '$mwdomain/analytics/';
593 \$wgPiwikIDSite = '1';
594 EOF
595 # </source>
596 # <source lang="bash">
597 s=~/bin/piwik-update
598 mkdir -p ${s%/*}
599 dd of=$s <<EOF
600 #!/bin/bash
601 piwik_path=$mw/../analytics
602 cd "\$(mktemp -d)"
603 wget -q http://builds.piwik.org/piwik.zip
604 unzip -q piwik.zip
605 rm -f piwik.zip
606 cp \$piwik_path/config/config.ini.php piwik/config
607 cp -rf piwik/* \$piwik_path
608 # prevent making an email out of the standard success response
609 x="\$( \$piwik_path/console core:update )"
610 if [[ \$x != "Everything is already up to date." ]]; then
611 echo "\$x"
612 fi
613 EOF
614 chmod +x $s
615
616 x="$(mktemp)"
617 crontab -l > "$x"
618 teeu "$x" <<< "0 3 * * tue $s"
619 crontab "$x"
620 # </source>