major update, working fully for mw 1.27
[mediawiki-setup] / mw-setup-script
1 #!/bin/bash
2 # Copyright (C) 2016 Ian Kelling
3 # This program is under GPL v. 3 or later, see <http://www.gnu.org/licenses/>
4 # <source lang="bash">
5 # for convenience, Mediawiki config file
6 mwc="$mw/LocalSettings.php"
7
8 # identify if this is a debian based distro
9 isdeb() { command -v apt &>/dev/null; }
10 # tee unique. append each stdin line if it does not exist in the file
11 teeu () {
12 local MAPFILE
13 mapfile -t
14 for line in "${MAPFILE[@]}"; do
15 grep -xFq "$line" "$1" &>/dev/null || tee -a "$1" <<<"$line"
16 done
17 }
18
19 # get and reset an extension/skin repository, and enable it
20 mw-ext () { mw-extra extensions $@; }
21 mw-skin() { mw-extra skins $@; }
22
23 mw-extra() {
24 local type=$1 # extension or skin
25 shift
26 local ext
27 for ext in "$@"; do
28 local original_pwd="$PWD"
29 # it's ok that this fails if we already have it
30 url=https://git.wikimedia.org/git/mediawiki
31 target=$mw/$type/$ext
32 if [[ ! -e $target/.git ]]; then
33 git clone $url/$type/$ext.git $target
34 fi
35 if ! cd $target; then
36 echo "mw-ext error: failed cd $mw/extensions/$ext";
37 exit 1
38 fi
39 git fetch
40 git checkout -qf origin/$mw_branch || git checkout -qf origin/master
41 git clean -xffd
42 cd "$original_pwd"
43 case $type in
44 extensions)
45 if [[ -e $target/extension.json ]]; then
46 # new style extension. remove old style declaration
47 sed -i '#^require_once( "\\\$IP/extensions/\$ext/\$ext\.php" );#d' $mwc
48 teeu $mwc <<EOF
49 wfLoadExtension( '$ext' );
50 EOF
51 else
52 teeu $mwc <<EOF
53 require_once( "\$IP/extensions/$ext/$ext.php" );
54 EOF
55 fi
56 ;;
57 skins)
58 sed -i '/^wfLoadSkin/d' $mwc
59 sed -i '/^\$wgDefaultSkin/d' $mwc
60 teeu $mwc <<EOF
61 \$wgDefaultSkin = "${ext,,*}";
62 wfLoadSkin( '$ext' );
63 EOF
64 ;;
65 esac
66 done
67 # --quick is quicker than default flags,
68 # but still add a sleep to make sure everything works right
69 sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
70 }
71
72 if command -v apt &>/dev/null; then
73 apache_user=www-data
74 else
75 apache_user=apache
76 fi
77
78 # </source>
79 # <source lang="bash">
80 # From here on out, exit if a command fails.
81 # This will prevent us from not noticing an important failure.
82 # We recommend setting this for the entire installation session.
83 # If you are running commands interactively, it might be best to
84 # put it in your ~/.bashrc temporarily.
85 set -eE -o pipefail
86 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
87 source ~/mw_vars
88
89 if isdeb; then
90 # main reference:
91 # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu
92 apt-get update
93 apt-get install -y ImageMagick
94 if apt-get install -s mediawiki &>/dev/null; then
95 # in debian wheezy time-frame distros, mediawiki was packaged.
96 apt-get -y install php-apc mediawiki
97 else
98 # https://www.mediawiki.org/wiki/Manual:Installation_requirements
99 if apt-get install -s php7.0 &>/dev/null; then
100 # note, 7.0 is untested by the editor here, since it's not
101 # available in debian 8. it's listed as supported
102 # in the mediawiki page.
103 # noninteractive to avoid mysql password prompt
104 DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 mysql-server \
105 php7.0 php7.0-mysql libapache2-mod-php7.0 php7.0-xml \
106 php7.0-apcu
107 else
108 DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 mysql-server \
109 php5 php5-mysql libapache2-mod-php5 php5-apcu
110 fi
111 fi
112 service apache2 restart
113 else
114 # note
115 # fedora deps are missing a database, so some is translated from debian packages
116 yum -y install mediawiki ImageMagick php-mysqlnd php-pecl-apcu mariadb-server
117
118 systemctl restart mariadb.service
119 systemctl enable mariadb.service
120 systemctl enable httpd.service
121 systemctl restart httpd.service
122 fi
123
124
125 # slightly different depending on if we already set the root pass
126 if echo exit|mysql -u root -p"$dbpass"; then
127 # answer interactive prompts:
128 # mysql root pass, change pass? no, remove anon users? (default, yes)
129 # disallow remote root (default, yes), reload? (default, yes)
130 echo -e "$dbpass\nn\n\n\n\n" | mysql_secure_installation
131 else
132 # I had 1 less newline at the start when doing ubuntu 14.04,
133 # compared to debian 8, so can't say this is especially portable.
134 # It won't hurt if it fails.
135 echo -e "\n\n$dbpass\n$dbpass\n\n\n\n\n" | mysql_secure_installation
136 fi
137 # </source>
138 # <source lang="bash">
139 mkdir -p $mw
140 cd $mw
141 # this will just fail if it already exists which is fine
142 if [[ ! -e .git ]]; then
143 git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
144 fi
145 # to see available branches: https://www.mediawiki.org/wiki/Version_lifecycle
146 # and
147 # git branch -r
148 git checkout -f origin/$mw_branch
149 git clean -ffxd
150 # Get the php libraries wmf uses. Based on:
151 # https://www.mediawiki.org/wiki/Download_from_Git#Fetch_external_libraries
152 if [[ ! -e vendor/.git ]]; then
153 git clone https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
154 fi
155 cd vendor
156 git checkout -f origin/$mw_branch
157 cd ..
158
159 # Drop any previous database which may have been installed while testing.
160 # If upgrading, we should have a db backup which will get restored.
161 # https://www.mediawiki.org/wiki/Manual:Upgrading
162 mysql -u root -p$dbpass <<'EOF' ||:
163 drop database my_wiki;
164 exit
165 EOF
166 php $mw/maintenance/install.php --pass $wikipass --scriptpath /w \
167 --dbuser root --dbpass $dbpass "$mwdescription" "$wikiuser"
168 teeu $mwc <<'EOF'
169 # lock down the wiki to only the initial owner until anti-spam measures are put in place
170 # limit edits to registered users
171 $wgGroupPermissions['*']['edit'] = false;
172 # don't allow any account creation
173 $wgGroupPermissions['*']['createaccount'] = false;
174 EOF
175 # </source>
176 # <source lang="bash">
177 tmpdir="$(mktemp -d)"
178 cd $tmpdir
179 wget http://builds.piwik.org/piwik.zip
180 if isdeb; then apt-get -y install unzip; else yum -y install unzip; fi
181 unzip -q piwik.zip
182 # gui installer suggested command
183 if isdeb; then
184 chown -R www-data:www-data piwik
185 else
186 chown -R apache:apache piwik
187 fi
188 # remove any existing directory
189 rm -rf $mw/../analytics
190 mv piwik $mw/../analytics
191 cd $mw
192 rm -rf $tmpdir
193
194 # </source>
195 # <source lang="bash">
196 temp=$(mktemp -d)
197 cd $temp
198 git_site=https://iankelling.org/git
199 git clone $git_site/acme-tiny-wrapper
200 l=$mw/../../logs
201 mkdir -p $l
202
203 acme-tiny-wrapper/acme-tiny-wrapper $mwdomain
204
205 git clone $git_site/basic-https-conf
206 { cat <<EOF
207 ServerAdmin $mw_email
208 RewriteEngine On
209 # make the site's root url go to our main page
210 RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
211 # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
212 RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
213 EOF
214 find -L $(readlink -f $mw/..) -name .htaccess \
215 | while read line; do
216 echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
217 done
218 } | basic-https-conf/apache-site -r ${mw%/*} - $mwdomain
219 # </source>
220 # <source lang="bash">
221 dd of=$mw/../robots.txt <<'EOF'
222 User-agent: *
223 Disallow: /w/
224 User-agent: ia_archiver
225 Allow: /*&action=raw
226 EOF
227 mw-skin Vector
228 # </source>
229 # <source lang="bash">
230 teeu $mwc<<EOF
231 \$wgServer = "https://$mwdomain";
232 \$wgDBserver = "localhost";
233 \$wgRightsUrl = "$mw_RightsUrl";
234 \$wgRightsText = "$mw_RightsText";
235 \$wgRightsIcon = "$mw_RightsIcon";
236 EOF
237 # </source>
238 # <source lang="bash">
239 teeu $mwc<<EOF
240 \$wgPasswordSender = "$mw_email";
241 \$wgEmergencyContact = "$mw_email";
242 \$wgEnotifUserTalk = true; # UPO
243 \$wgEnotifWatchlist = true; # UPO
244 \$wgMainCacheType = CACHE_ACCEL;
245 \$wgEnableUploads = true;
246 \$wgUseInstantCommons = true;
247 EOF
248 # </source>
249 # <source lang="bash">
250 teeu $mwc <<'EOF'
251 # from https://www.mediawiki.org/wiki/Manual:Short_URL
252 $wgArticlePath = "/wiki/$1";
253
254 # https://www.mediawiki.org/wiki/Manual:Combating_spam
255 # check that url if our precautions don't work
256 # not using nofollow is good practice, as long as we avoid spam.
257 $wgNoFollowLinks = false;
258 # Allow user customization.
259 $wgAllowUserJs = true;
260 $wgAllowUserCss = true;
261
262 # use imagemagick over GD
263 $wgUseImageMagick = true;
264 EOF
265
266
267 # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads
268 # Increase from default of 2M to 100M.
269 # This will at least allow high res pics etc.
270 php_ini=$(isdeb && echo /etc/php5/apache2/php.ini || echo /etc/php.ini)
271 sed -i 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini
272 if isdeb; then
273 service apache2 restart
274 else
275 systemctl restart httpd.service
276 fi
277
278 # if you were to install as a normal user, you would need this for images
279 # sudo usermod -aG $apache_user $USER
280
281 # this doesn't propogate right away
282 chgrp -R $apache_user $mw/images
283 chmod -R g+w $mw/images
284 # </source>
285 # <source lang="bash">
286 teeu $mwc <<'EOF'
287 $wgLogo = null;
288 #$wgFooterIcons = null;
289 EOF
290 # Make the toolbox go into the drop down.
291 cd $mw/skins/Vector
292 if ! git remote show ian-kelling &>/dev/null; then
293 git remote add ian-kelling https://iankelling.org/git/Vector
294 fi
295 git fetch ian-kelling
296 git checkout ian-kelling/REL1_27-toolbox-in-dropdown
297 # </source>
298 # <source lang="bash">
299 mw-ext Cite CiteThisPage CSS Echo Gadgets ImageMap Interwiki News \
300 Nuke ParserFunctions Poem SyntaxHighlight_GeSHi Variables
301 # </source>
302 # <source lang="bash">
303 mw-ext AntiSpoof
304 # recommended setup script to account for existing users
305 sudo -u $apache_user php $mw/extensions/AntiSpoof/maintenance/batchAntiSpoof.php
306 # </source>
307 # <source lang="bash">
308 mw-ext CheckUser
309 sudo -u $apache_user php $mw/extensions/CheckUser/install.php; sleep 1
310 # </source>
311 # <source lang="bash">
312 if isdeb; then
313 apt-get -y install php-wikidiff2
314 teeu $mwc <<'EOF'
315 $wgExternalDiffEngine = 'wikidiff2';
316 EOF
317 ln -sf ../../mods-available/wikidiff2.ini /etc/php5/apache2/conf.d
318 service apache2 restart
319 fi
320 # </source>
321 # <source lang="bash">
322 mw-ext Math
323 # php5-curl according to Math readme
324 if isdeb; then
325 curl_pkg=php7.0-curl
326 if ! apt-get -s install $curl_pkg &>/dev/null; then
327 curl_pkg=php5-curl
328 fi
329 apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base \
330 ghostscript imagemagick ocaml $curl_pkg make
331 else
332 # todo, php5-curl equivalent on fedora
333 yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml
334 fi
335 ln -sf ../../mods-available/curl.ini /etc/php5/apache2/conf.d
336 service apache2 restart
337
338 cd $mw/extensions/Math/math; make # makes texvc
339 cd $mw/extensions/Math/texvccheck; make
340
341 teeu $mwc <<'EOF'
342 # Enable MathJax as rendering option
343 $wgUseMathJax = true;
344 # Enable LaTeXML as rendering option
345 $wgMathValidModes[] = 'latexml';
346 # Set LaTeXML as default rendering option, because it is nicest
347 $wgDefaultUserOptions['math'] = 'latexml';
348 EOF
349 # </source>
350 # <source lang="bash">
351 mw-ext SpamBlacklist
352 if ! grep -F '$wgSpamBlacklistFiles = array(' $mwc &>/dev/null; then
353 tee -a $mwc <<'EOF'
354 $wgEnableDnsBlacklist = true;
355 $wgDnsBlacklistUrls = array( 'xbl.spamhaus.org', 'dnsbl.tornevall.org' );
356
357 ini_set( 'pcre.backtrack_limit', '10M' );
358 $wgSpamBlacklistFiles = array(
359 "[[m:Spam blacklist]]",
360 "http://en.wikipedia.org/wiki/MediaWiki:Spam-blacklist"
361 );
362 EOF
363 fi
364 # </source>
365 # <source lang="bash">
366 mw-ext TitleBlacklist
367 if ! grep -F '$wgTitleBlacklistSources = array(' $mwc &>/dev/null; then
368 tee -a $mwc <<'EOF'
369 $wgTitleBlacklistSources = array(
370 array(
371 'type' => 'local',
372 'src' => 'MediaWiki:Titleblacklist',
373 ),
374 array(
375 'type' => 'url',
376 'src' => 'http://meta.wikimedia.org/w/index.php?title=Title_blacklist&action=raw',
377 ),
378 );
379 EOF
380 fi
381 # </source>
382 # <source lang="bash">
383 mw-ext WikiEditor
384 teeu $mwc <<'EOF'
385 # Enable Wikieditor by default
386 $wgDefaultUserOptions['usebetatoolbar'] = 1;
387 $wgDefaultUserOptions['usebetatoolbar-cgd'] = 1;
388
389 # Display the Preview and Changes tabs
390 $wgDefaultUserOptions['wikieditor-preview'] = 1;
391 EOF
392 # </source>
393 # <source lang="bash">
394 mw-ext CategoryTree
395 teeu $mwc <<'EOF'
396 # Mediawiki setting dependency for CategoryTree
397 $wgUseAjax = true;
398 EOF
399 # </source>
400 # <source lang="bash">
401 mw-ext AbuseFilter
402 teeu $mwc<<'EOF'
403 $wgGroupPermissions['sysop']['abusefilter-modify'] = true;
404 $wgGroupPermissions['*']['abusefilter-log-detail'] = true;
405 $wgGroupPermissions['*']['abusefilter-view'] = true;
406 $wgGroupPermissions['*']['abusefilter-log'] = true;
407 $wgGroupPermissions['sysop']['abusefilter-private'] = true;
408 $wgGroupPermissions['sysop']['abusefilter-modify-restricted'] = true;
409 $wgGroupPermissions['sysop']['abusefilter-revert'] = true;
410 EOF
411 # </source>
412 # <source lang="bash">
413 mw-ext ConfirmEdit
414 captchaArray
415 teeu $mwc <<'EOF'
416 wfLoadExtension( 'ConfirmEdit/QuestyCaptcha' );
417 $wgCaptchaClass = 'QuestyCaptcha';
418 # only captcha on registration
419 $wgGroupPermissions['user' ]['skipcaptcha'] = true;
420 $wgGroupPermissions['autoconfirmed']['skipcaptcha'] = true;
421 EOF
422 if ! grep -Fx 'foreach ( $localSettingsQuestyQuestions as $key => $value ) {' $mwc; then
423 tee -a $mwc <<'EOF'
424 foreach ( $localSettingsQuestyQuestions as $key => $value ) {
425 $wgCaptchaQuestions[] = array( 'question' => $key, 'answer' => $value );
426 }
427 EOF
428 fi
429 # </source>
430 # <source lang="bash">
431 sed -i "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
432 # </source>
433 # <source lang="bash">
434 # get repo
435 if [[ ! -e ~/pywikibot/.git ]]; then
436 git clone --recursive \
437 https://gerrit.wikimedia.org/r/pywikibot/core.git ~/pywikibot
438 fi
439 cd ~/pywikibot
440 #updating
441 git pull --all
442 git submodule update
443 # </source>
444 # <source lang="bash">
445 cd $HOME/pywikibot
446 dd of=user-config.py <<EOF
447 mylang = 'en'
448 usernames["$mwfamily"]['en'] = u'$wikiuser'
449 family = "$mwfamily"
450 console_encoding = 'utf-8'
451 password_file = "secretsfile"
452 EOF
453
454 dd of=secretsfile <<EOF
455 ("$wikiuser", "$wikipass")
456 EOF
457
458 # it won't overrwrite an existing file. Remove if if one exists
459 rm -f pywikibot/families/${mwfamily}_family.py
460 if isdeb; then
461 apt-get install -y python-requests
462 else
463 yum -y install python-requests
464 fi
465
466 python generate_family_file.py https://$mwdomain/wiki/Main_Page "$mwfamily"
467
468 # Note, this needed only for ssl site
469 tee -a pywikibot/families/${mwfamily}_family.py<<'EOF'
470 def protocol(self, code):
471 return 'https'
472 EOF
473 # </source>
474 # <source lang="bash">
475 cd "$HOME/pywikibot"
476
477 dd of=scripts/${mwfamily}_setup.py<<EOF
478 import pywikibot
479 import time
480 import sys
481 site = pywikibot.Site()
482 def x(p, t=""):
483 page = pywikibot.Page(site, p)
484 page.text = t
485 #force is for some anti-bot thing, not necessary in my testing, but might as well include it
486 page.save(force=True)
487
488 # Small/medium noncommercial wiki should be fine with no privacy policy
489 # based on https://www.mediawiki.org/wiki/Manual:Footer
490 x("MediaWiki:Privacy")
491
492 # licenses for uploads. Modified from the mediawiki's wiki
493 x("MediaWiki:Licenses", u"""* Same as this wiki's text (preferred)
494 ** CC BY-SA or GFDL| Creative Commons Attribution ShareAlike or GNU Free Documentation License
495 * Others:
496 ** Unknown_copyright|I don't know exactly
497 ** PD|PD: public domain
498 ** CC BY|Creative Commons Attribution
499 ** CC BY-SA|Creative Commons Attribution ShareAlike
500 ** GFDL|GFDL: GNU Free Documentation License
501 ** GPL|GPL: GNU General Public License
502 ** LGPL|LGPL: GNU Lesser General Public License""")
503 x("MediaWiki:Copyright", '$mw_license')
504 x("MediaWiki:Mainpage-description", "$mwdescription")
505
506
507
508 # The rest of the settings are for the site style
509
510 # Remove various clutter
511 x("MediaWiki:Lastmodifiedat")
512 x("MediaWiki:Disclaimers")
513 x("MediaWiki:Viewcount")
514 x("MediaWiki:Aboutsite")
515 # remove these lines from sidebar
516 # ** recentchanges-url|recentchanges
517 # ** randompage-url|randompage
518 # ** helppage|help
519 x("MediaWiki:Sidebar", """* navigation
520 ** mainpage|mainpage-description
521 * SEARCH
522 * TOOLBOX
523 * LANGUAGES""")
524
525 # remove side panel
526 # helpfull doc: https://www.mediawiki.org/wiki/Manual:Interface/Sidebar
527 x("mediawiki:Common.css", """/* adjust sidebar to just be home link and up top */
528 /* panel width increased to fit full wiki name. */
529 div#mw-panel { top: 10px; padding-top: 0em; width: 20em }
530 div#footer, #mw-head-base, div#content { margin-left: 1em; }
531 #left-navigation { margin-left: 1em; }
532
533
534 /* logo, and toolbar hidden */
535 #p-logo, #p-tb.portal {
536 display:none;
537 }
538
539 /* make the font size smaller for the misc stuff */
540 #p-personal {
541 font-size: 0.8em;
542 }
543
544 #footer-info {
545 font-size: 0.8em;
546 }
547 div#mw-content-text {
548 max-width: 720px;
549 }
550 """)
551 EOF
552
553 python pwb.py ${mwfamily}_setup
554 # </source>
555 # <source lang="bash">
556 s=/etc/cron.daily/mediawiki_update
557 dd of=$s<<'EOF'
558 #!/bin/bash
559 source ~/mw_vars
560 cd $mw
561 git fetch --all
562 git checkout origin/$mw_branch
563 git rebase ian/REL1_23-toolbox-in-dropdown
564 cd extensions
565 for x in *; do
566 if [[ -d $x ]]; then
567 cd $x
568 git fetch --all
569 git checkout origin/$mw_branch || git checkout -qf origin/master
570 cd ..
571 fi
572 done
573 php $mw/maintenance/update.php -q
574 EOF
575 chmod +x $s
576 # </source>
577 # <source lang="bash">
578 # based on recommendation from install page
579 if ! dpkg -s php5-gd &>/dev/null; then
580 apt-get install -y php5-gd
581 service apache2 restart
582 fi
583
584 # docs suggests using separate database user
585 mysql -u root -p$dbpass <<EOF
586 GRANT ALL ON piwik.* TO 'piwik'@'localhost' IDENTIFIED BY '$piwik_pass';
587 FLUSH PRIVILEGES;
588 exit
589 EOF
590 php_ini=$(isdeb && echo /etc/php5/apache2/php.ini || echo /etc/php.ini)
591 # based on the install page. however, ths option is changing with php7.0
592 opt=always_populate_raw_post_data
593 sed -ri "/^ *$opt\b/d;/^ *\[PHP\]/a $opt = -1" $php_ini
594 service apache2 restart
595 # </source>
596 # <source lang="bash">
597 git clone https://github.com/DaSchTour/piwik-mediawiki-extension.git $mw/extensions/Piwik
598 mw-ext Piwik
599 teeu $mwc <<EOF
600 \$wgPiwikURL = '$mwdomain/analytics/';
601 \$wgPiwikIDSite = '1';
602 EOF
603 # </source>