retire it for now
[mediawiki-setup] / mw-setup-script
1 #!/bin/bash
2 # Copyright (C) 2016 Ian Kelling
3 # This program is under GPL v. 3 or later, see <http://www.gnu.org/licenses/>
4 set -x
5 # <source lang="bash">
6 # if we have jessie-backports, we need to use it.
7 if ! apt-get install -t jessie-backports certbot python-certbot-apache; then
8 apt-get install certbot python-certbot-apache
9 fi
10 # </source>
11 # <source lang="bash">
12 # identify if this is a debian based distro
13 isdeb() { command -v apt &>/dev/null; }
14 # tee unique. append each stdin line if it does not exist in the file
15 teeu () {
16 local MAPFILE
17 mapfile -t
18 for line in "${MAPFILE[@]}"; do
19 grep -xFq "$line" "$1" &>/dev/null || tee -a "$1" <<<"$line"
20 done
21 }
22
23 # get and reset an extension/skin repository, and enable it
24 mw-clone() {
25 local url=$1
26 local original_pwd="$PWD"
27 local name
28 local re='[^/]*/[^/]*$' # last 2 parts of path
29 [[ $url =~ $re ]] ||:
30 target=$mw/${BASH_REMATCH[0]}
31 if [[ ! -e $target/.git ]]; then
32 git clone $url $target
33 fi
34 if ! cd $target; then
35 echo "mw-ext error: failed cd $target";
36 exit 1
37 fi
38 git fetch
39 git checkout -qf origin/$mw_branch || git checkout -qf origin/master
40 git clean -xffd
41 cd "$original_pwd"
42
43 }
44 mw-ext () {
45 local ext
46 for ext; do
47 mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/$ext
48 if [[ -e $mw/extensions/$ext/extension.json ]]; then
49 # new style extension
50 teeu $mwc <<EOF
51 wfLoadExtension( '$ext' );
52 EOF
53 else
54 teeu $mwc <<EOF
55 require_once( "\$IP/extensions/$ext/$ext.php" );
56 EOF
57 fi
58 done
59 # --quick is quicker than default flags,
60 # but still add a sleep to make sure everything works right
61 sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
62 }
63 mw-skin() {
64 local skin=$1
65 mw-clone https://gerrit.wikimedia.org/r/p/mediawiki/skins/$skin
66 sed -i --follow-symlinks '/^wfLoadSkin/d' $mwc
67 sed -i --follow-symlinks '/^\$wgDefaultSkin/d' $mwc
68 teeu $mwc <<EOF
69 \$wgDefaultSkin = "${skin,,*}";
70 wfLoadSkin( '$skin' );
71 EOF
72 sudo -u $apache_user php $mw/maintenance/update.php -q --quick; sleep 1
73 }
74
75 if command -v apt &>/dev/null; then
76 apache_user=www-data
77 else
78 apache_user=apache
79 fi
80
81 # </source>
82 # <source lang="bash">
83 # From here on out, exit if a command fails.
84 # This will prevent us from not noticing an important failure.
85 # We recommend setting this for the entire installation session.
86 # If you are running commands interactively, it might be best to
87 # put it in your ~/.bashrc temporarily.
88 set -eE -o pipefail
89 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
90 source ~/mw_vars
91
92 if isdeb; then
93 # main reference:
94 # https://www.mediawiki.org/wiki/Manual:Running_MediaWiki_on_Ubuntu
95 apt-get update
96 DEBIAN_FRONTEND=noninteractive apt-get install -y imagemagick curl
97 if apt-get install -s mediawiki &>/dev/null; then
98 apcu=php5-apcu
99 if apt-get install -s php7.0-apcu &>/dev/null; then
100 apcu=php7.0-apcu
101 fi
102 # mediawiki is packaged in jessie backports & stretch
103 DEBIAN_FRONTEND=noninteractive apt-get -y install $apcu mediawiki
104 else
105 # https://www.mediawiki.org/wiki/Manual:Installation_requirements
106 if apt-get install -s php7.0 &>/dev/null; then
107 # note, 7.0 is untested by the editor here, since it's not
108 # available in debian 8. it's listed as supported
109 # in the mediawiki page.
110 # noninteractive to avoid mysql password prompt.
111 DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 \
112 default-mysql-server \
113 php7.0 php7.0-mysql libapache2-mod-php7.0 php7.0-xml \
114 php7.0-apcu php7.0-mbstring
115 else
116 # note: mbstring is recommended, but it's not available for php5 in
117 # debian jessie.
118 DEBIAN_FRONTEND=noninteractive apt-get install -y apache2 \
119 default-mysql-server \
120 php5 php5-mysql libapache2-mod-php5 php5-apcu
121 fi
122 fi
123 service apache2 restart
124 else
125 # note
126 # fedora deps are missing a database, so some is translated from debian packages
127 yum -y install mediawiki ImageMagick php-mysqlnd php-pecl-apcu mariadb-server
128
129 systemctl restart mariadb.service
130 systemctl enable mariadb.service
131 systemctl enable httpd.service
132 systemctl restart httpd.service
133 fi
134
135
136 # skip if we already set the root pass and are on pre-debian 9.
137 if ! echo exit|mysql -uroot "-p$dbpass"; then
138 # Note: we set a root password here, but in debian 9+, it is ignored;
139 # only the local user root can login, and any password is accepted.
140 # We answer these interactive prompts:
141 # Enter current password for root (enter for none):
142 # Set root password? [Y/n]
143 # New password:
144 # Re-enter new password:
145 # Remove anonymous users? [Y/n]
146 # Disallow root login remotely? [Y/n]
147 # Remove test database and access to it? [Y/n]
148 # Reload privilege tables now? [Y/n]
149 # Note, I had 1 less newline at the start when doing ubuntu 14.04,
150 # compared to debian 8, so can't say this is especially portable.
151 echo -e "\n\n$dbpass\n$dbpass\n\n\n\n\n" | mysql_secure_installation
152 fi
153 mysql -uroot "-p$dbpass" <<EOF
154 GRANT ALL PRIVILEGES ON my_wiki.* TO 'wikiuser'@'localhost' IDENTIFIED BY '$dbpass';
155 EOF
156 # </source>
157 # <source lang="bash">
158 mkdir -p $mw
159 cd $mw
160 # this will just fail if it already exists which is fine
161 if [[ ! -e .git ]]; then
162 git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git .
163 fi
164 # to see available branches: https://www.mediawiki.org/wiki/Version_lifecycle
165 # and
166 # git branch -r
167 git checkout -f origin/$mw_branch
168 git clean -ffxd
169 # apply librejs patch
170 curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-1.28-librejs.patch;hb=HEAD" | patch -r - -N -p1
171 # Get the php libraries wmf uses. Based on:
172 # https://www.mediawiki.org/wiki/Download_from_Git#Fetch_external_libraries
173 if [[ ! -e vendor/.git ]]; then
174 git clone https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
175 fi
176 cd vendor
177 git checkout -f origin/$mw_branch
178 cd ..
179
180 # Drop any previous database which may have been installed while testing.
181 # If upgrading, we should have a db backup which will get restored.
182 # https://www.mediawiki.org/wiki/Manual:Upgrading
183 mysql -uroot "-p$dbpass" <<'EOF' ||:
184 drop database my_wiki;
185 exit
186 EOF
187 php $mw/maintenance/install.php --pass $wikipass --scriptpath /w \
188 --dbuser wikiuser --dbpass $dbpass "$mwdescription" "$wikiuser"
189 teeu $mwc <<'EOF'
190 # lock down the wiki to only the initial owner until anti-spam measures are put in place
191 # limit edits to registered users
192 $wgGroupPermissions['*']['edit'] = false;
193 # don't allow any account creation
194 $wgGroupPermissions['*']['createaccount'] = false;
195 EOF
196 # </source>
197 # <source lang="bash">
198 l=$mw/../../logs
199 mkdir -p $l
200 temp=$(mktemp -d)
201 cd $temp
202 # for me, this repo is on the same server and apache needs a sec after restarting
203 sleep 1
204 git clone https://iankelling.org/git/basic-https-conf
205 { cat <<EOF
206 ServerAdmin $mw_email
207 RewriteEngine On
208 # make the site's root url go to our main page
209 RewriteRule ^/?wiki(/.*)?\$ %{DOCUMENT_ROOT}/w/index.php [L]
210 # use short urls https://www.mediawiki.org/wiki/Manual:Short_URL
211 RewriteRule ^/*\$ %{DOCUMENT_ROOT}/w/index.php [L]
212 EOF
213 find -L $(readlink -f $mw) -name .htaccess \
214 | while read line; do
215 echo -e "<Directory ${line%/.htaccess}>\n $(< $line)\n</Directory>";
216 done
217 } | basic-https-conf/web-conf -r ${mw%/*} - apache2 $mwdomain
218 cd
219 rm -rf $temp
220 # </source>
221 # <source lang="bash">
222 dd of=$mw/../robots.txt <<'EOF'
223 User-agent: *
224 Disallow: /w/
225 User-agent: ia_archiver
226 Allow: /*&action=raw
227 EOF
228 mw-skin Vector
229 # </source>
230 # <source lang="bash">
231 teeu $mwc<<EOF
232 \$wgServer = "https://$mwdomain";
233 \$wgDBserver = "localhost";
234 \$wgRightsUrl = "$mw_RightsUrl";
235 \$wgRightsText = "$mw_RightsText";
236 \$wgRightsIcon = "$mw_RightsIcon";
237 EOF
238 # </source>
239 # <source lang="bash">
240 teeu $mwc<<EOF
241 \$wgPasswordSender = "$mw_email";
242 \$wgEmergencyContact = "$mw_email";
243 \$wgEnotifUserTalk = true; # UPO
244 \$wgEnotifWatchlist = true; # UPO
245 \$wgMainCacheType = CACHE_ACCEL;
246 \$wgEnableUploads = true;
247 \$wgUseInstantCommons = true;
248 \$wgPingback = true;
249 EOF
250 # </source>
251 # <source lang="bash">
252 teeu $mwc <<'EOF'
253 # from https://www.mediawiki.org/wiki/Manual:Short_URL
254 $wgArticlePath = "/wiki/$1";
255
256 # https://www.mediawiki.org/wiki/Manual:Combating_spam
257 # check that url if our precautions don't work
258 # not using nofollow is good practice, as long as we avoid spam.
259 $wgNoFollowLinks = false;
260 # Allow user customization.
261 $wgAllowUserCss = true;
262 # use imagemagick over GD
263 $wgUseImageMagick = true;
264 # manual says this is not production ready, I think that is mostly
265 # because they are using MobileFrontend extension instead, which gives
266 # an even cleaner more minimal view, I plan to try setting it up
267 # sometime but this seems like a very nice improvement for now.
268 $wgVectorResponsive = true;
269 EOF
270
271
272 # https://www.mediawiki.org/wiki/Manual:Configuring_file_uploads
273 # Increase from default of 2M to 100M.
274 # This will at least allow high res pics etc.
275 php_ini=$(php -r 'echo(php_ini_loaded_file());')
276 sed -i --follow-symlinks 's/^\(upload_max_filesize\|post_max_size\)\b.*/\1 = 100M/' $php_ini
277 if isdeb; then
278 service apache2 restart
279 else
280 systemctl restart httpd.service
281 fi
282
283 # if you were to install as a normal user, you would need this for images
284 # sudo usermod -aG $apache_user $USER
285
286 # this doesn't propogate right away
287 chgrp -R $apache_user $mw/images
288 chmod -R g+w $mw/images
289 # </source>
290 # <source lang="bash">
291 teeu $mwc <<'EOF'
292 $wgLogo = null;
293 #$wgFooterIcons = null;
294 EOF
295 # Make the toolbox go into the drop down.
296 cd $mw/skins/Vector
297 if ! git remote show ian-kelling &>/dev/null; then
298 git remote add ian-kelling https://iankelling.org/git/mediawiki-sidebar-patch
299 fi
300 git fetch ian-kelling
301 git checkout ian-kelling/${mw_branch}-toolbox-in-dropdown
302 # </source>
303 # <source lang="bash">
304 mw-ext Cite CiteThisPage CheckUser CSS Echo Gadgets ImageMap Interwiki News \
305 Nuke ParserFunctions Poem Renameuser SyntaxHighlight_GeSHi Variables
306 # </source>
307 # <source lang="bash">
308 mw-ext AntiSpoof
309 # recommended setup script to account for existing users
310 sudo -u $apache_user php $mw/extensions/AntiSpoof/maintenance/batchAntiSpoof.php
311 # </source>
312 # <source lang="bash">
313 if isdeb; then
314 apt-get -y install php-wikidiff2
315 teeu $mwc <<'EOF'
316 $wgExternalDiffEngine = 'wikidiff2';
317 EOF
318 dir=$(dirname $(php -r 'echo(php_ini_loaded_file());'))/../apache2/conf.d
319 ln -sf ../../mods-available/wikidiff2.ini $dir
320 service apache2 restart
321 fi
322 # </source>
323 # <source lang="bash">
324 mw-ext Math
325 # php5-curl according to Math readme
326 if isdeb; then
327 curl_pkg=php7.0-curl
328 if ! apt-get -s install $curl_pkg &>/dev/null; then
329 curl_pkg=php5-curl
330 fi
331 apt-get -y install latex-cjk-all texlive-latex-extra texlive-latex-base \
332 ghostscript imagemagick ocaml $curl_pkg make
333 else
334 # todo, php5-curl equivalent on fedora
335 yum -y install texlive-cjk ghostscript ImageMagick texlive ocaml
336 fi
337 service apache2 restart
338
339 cd $mw/extensions/Math/math; make # makes texvc
340 cd $mw/extensions/Math/texvccheck; make
341
342 teeu $mwc <<'EOF'
343 # Enable MathJax as rendering option
344 $wgUseMathJax = true;
345 # Enable LaTeXML as rendering option
346 $wgMathValidModes[] = 'latexml';
347 # Set LaTeXML as default rendering option, because it is nicest
348 $wgDefaultUserOptions['math'] = 'latexml';
349 EOF
350 # </source>
351 # <source lang="bash">
352 mw-ext SpamBlacklist
353 if ! grep -F '$wgSpamBlacklistFiles = array(' $mwc &>/dev/null; then
354 tee -a $mwc <<'EOF'
355 $wgEnableDnsBlacklist = true;
356 $wgDnsBlacklistUrls = array( 'xbl.spamhaus.org', 'dnsbl.tornevall.org' );
357
358 ini_set( 'pcre.backtrack_limit', '10M' );
359 $wgSpamBlacklistFiles = array(
360 "[[m:Spam blacklist]]",
361 "http://en.wikipedia.org/wiki/MediaWiki:Spam-blacklist"
362 );
363 EOF
364 fi
365 # </source>
366 # <source lang="bash">
367 mw-ext TitleBlacklist
368 if ! grep -F '$wgTitleBlacklistSources = array(' $mwc &>/dev/null; then
369 tee -a $mwc <<'EOF'
370 $wgTitleBlacklistSources = array(
371 array(
372 'type' => 'local',
373 'src' => 'MediaWiki:Titleblacklist',
374 ),
375 array(
376 'type' => 'url',
377 'src' => 'http://meta.wikimedia.org/w/index.php?title=Title_blacklist&action=raw',
378 ),
379 );
380 EOF
381 fi
382 # </source>
383 # <source lang="bash">
384 mw-ext WikiEditor
385 teeu $mwc <<'EOF'
386 # Enable Wikieditor by default
387 $wgDefaultUserOptions['usebetatoolbar'] = 1;
388 $wgDefaultUserOptions['usebetatoolbar-cgd'] = 1;
389
390 # Display the Preview and Changes tabs
391 $wgDefaultUserOptions['wikieditor-preview'] = 1;
392 EOF
393 # </source>
394 # <source lang="bash">
395 mw-ext CategoryTree
396 teeu $mwc <<'EOF'
397 # Mediawiki setting dependency for CategoryTree
398 $wgUseAjax = true;
399 EOF
400 # </source>
401 # <source lang="bash">
402 mw-ext AbuseFilter
403 teeu $mwc<<'EOF'
404 $wgGroupPermissions['sysop']['abusefilter-modify'] = true;
405 $wgGroupPermissions['*']['abusefilter-log-detail'] = true;
406 $wgGroupPermissions['*']['abusefilter-view'] = true;
407 $wgGroupPermissions['*']['abusefilter-log'] = true;
408 $wgGroupPermissions['sysop']['abusefilter-private'] = true;
409 $wgGroupPermissions['sysop']['abusefilter-modify-restricted'] = true;
410 $wgGroupPermissions['sysop']['abusefilter-revert'] = true;
411 EOF
412 # </source>
413 # <source lang="bash">
414 mw-ext ConfirmEdit
415 captchaArray
416 teeu $mwc <<'EOF'
417 wfLoadExtension( 'ConfirmEdit/QuestyCaptcha' );
418 $wgCaptchaClass = 'QuestyCaptcha';
419 # only captcha on registration
420 $wgGroupPermissions['user' ]['skipcaptcha'] = true;
421 $wgGroupPermissions['autoconfirmed']['skipcaptcha'] = true;
422 EOF
423 if ! grep -Fx 'foreach ( $localSettingsQuestyQuestions as $key => $value ) {' $mwc; then
424 tee -a $mwc <<'EOF'
425 foreach ( $localSettingsQuestyQuestions as $key => $value ) {
426 $wgCaptchaQuestions[] = array( 'question' => $key, 'answer' => $value );
427 }
428 EOF
429 fi
430 # </source>
431 # <source lang="bash">
432 sed -i --follow-symlinks "/\\\$wgGroupPermissions\\['\\*'\\]\\['createaccount'\\] = false;/d" $mwc
433 # </source>
434 # <source lang="bash">
435 # get repo
436 if [[ ! -e ~/pywikibot/.git ]]; then
437 git clone --recursive \
438 https://gerrit.wikimedia.org/r/pywikibot/core.git ~/pywikibot
439 fi
440 cd ~/pywikibot
441 #updating
442 git pull --all
443 git submodule update
444 # </source>
445 # <source lang="bash">
446 cd $HOME/pywikibot
447 dd of=user-config.py <<EOF
448 mylang = 'en'
449 usernames["$mwfamily"]['en'] = u'$wikiuser'
450 family = "$mwfamily"
451 console_encoding = 'utf-8'
452 password_file = "secretsfile"
453 EOF
454
455 dd of=secretsfile <<EOF
456 ("$wikiuser", "$wikipass")
457 EOF
458
459 # it won't overrwrite an existing file. Remove if if one exists
460 rm -f pywikibot/families/${mwfamily}_family.py
461 if isdeb; then
462 apt-get install -y python-requests
463 else
464 yum -y install python-requests
465 fi
466
467 python generate_family_file.py https://$mwdomain/wiki/Main_Page "$mwfamily"
468
469 # Note, this needed only for ssl site
470 tee -a pywikibot/families/${mwfamily}_family.py<<'EOF'
471 def protocol(self, code):
472 return 'https'
473 EOF
474 # </source>
475 # <source lang="bash">
476 cd "$HOME/pywikibot"
477
478 dd of=scripts/${mwfamily}_setup.py<<EOF
479 import pywikibot
480 import time
481 import sys
482 site = pywikibot.Site()
483 def x(p, t=""):
484 page = pywikibot.Page(site, p)
485 page.text = t
486 #force is for some anti-bot thing, not necessary in my testing, but might as well include it
487 page.save(force=True)
488
489 # Small/medium noncommercial wiki should be fine with no privacy policy
490 # based on https://www.mediawiki.org/wiki/Manual:Footer
491 x("MediaWiki:Privacy")
492
493 # licenses for uploads. Modified from the mediawiki's wiki
494 x("MediaWiki:Licenses", u"""* Same as this wiki's text (preferred)
495 ** CC BY-SA or GFDL| Creative Commons Attribution ShareAlike or GNU Free Documentation License
496 * Others:
497 ** Unknown_copyright|I don't know exactly
498 ** PD|PD: public domain
499 ** CC BY|Creative Commons Attribution
500 ** CC BY-SA|Creative Commons Attribution ShareAlike
501 ** GFDL|GFDL: GNU Free Documentation License
502 ** GPL|GPL: GNU General Public License
503 ** LGPL|LGPL: GNU Lesser General Public License""")
504 x("MediaWiki:Copyright", '$mw_license')
505 x("MediaWiki:Mainpage-description", "$mwdescription")
506
507
508
509 # The rest of the settings are for the site style
510
511 # Remove various clutter
512 x("MediaWiki:Lastmodifiedat")
513 x("MediaWiki:Disclaimers")
514 x("MediaWiki:Viewcount")
515 x("MediaWiki:Aboutsite")
516 # remove these lines from sidebar
517 # ** recentchanges-url|recentchanges
518 # ** randompage-url|randompage
519 # ** helppage|help
520 x("MediaWiki:Sidebar", """* navigation
521 ** mainpage|mainpage-description
522 * SEARCH
523 * TOOLBOX
524 * LANGUAGES""")
525
526 # remove side panel
527 # helpfull doc: https://www.mediawiki.org/wiki/Manual:Interface/Sidebar
528 x("mediawiki:Common.css", """/* adjust sidebar to just be home link and up top */
529 /* adjust sidebar to just be home link and up top */
530 /* panel width increased to fit full wiki name. */
531 /* selectors other than final id are for increasing priority of rule */
532 div#mw-panel { top: 10px; padding-top: 0em; width: 20em }
533 div#footer, #mw-head-base, div#content { margin-left: 1em; }
534 #left-navigation { margin-left: 1em; }
535
536
537 /* logo, and toolbar hidden */
538 #p-logo, div#mw-navigation div#mw-panel #p-tb {
539 display:none;
540 }
541
542 div#mw-content-text {
543 max-width: 720px;
544 }
545 """)
546 EOF
547
548 # this can spam a warning, so uniq it
549 python pwb.py ${mwfamily}_setup |& uniq
550 # </source>
551 # <source lang="bash">
552 s=/etc/cron.daily/mediawiki_update
553 dd of=$s<<'EOF'
554 #!/bin/bash
555 source ~/mw_vars
556 update() {
557 dir=$1
558 cd $mw
559 [[ -d $dir ]] || return 1
560 cd $dir
561 branch=$(git describe --all)
562 branch=${branch#remotes/}
563 git fetch --all -q
564 new_head=$(git rev-parse $branch)
565 log=$(git log HEAD..$new_head)
566 if [[ ! $log ]]; then
567 return 1
568 fi
569 pwd
570 echo "$log"
571 git checkout -qf $new_head
572 cd $mw
573 return 0
574 }
575 for dir in extensions/* skins/* vendor; do
576 update "$dir" ||:
577 done
578 if update .; then
579 curl "https://iankelling.org/git/?p=mediawiki-librejs-patch;a=blob_plain;f=mediawiki-1.28-librejs.patch;hb=HEAD" | patch -r - -N -p1
580 fi
581 php $mw/maintenance/update.php -q --quick
582 EOF
583
584 # </source>