add redirect www.clarissariviere.fr

Reviewed-on: #3

inventory/host_vars/vps-host

@@ -1,30 +1,36 @@
 $ANSIBLE_VAULT;1.1;AES256
-33636162336330363833666465326430326230353032643365623263306139346138363135316462
-6232623366616434333833333630663435333237306563630a333065363335653361613135316131
-38346464653533633062636534303937366534383064376232336635663665323163386566336465
-3037636164626361390a373963346334616232323639353561306631333834613964363635626330
-62633139383230373063313136383534653230323038313762323430393164616534363836376333
-30303662613534333631393031303165376435363831323636316561336530393963313465356164
-37386462643361363265326230393465313536343533646437393634663663663862643033323966
-34396162613134343738343861666663643338363136383262666263623933316435323135656331
-30373134376565633662646365636464663562636666303061653664633138326139373737303163
-30613561366233333264336562633837393239626462633238323564386332613861323266643733
-37653764646637313631326365373935313762393934663234326362393762313434663366663165
-62313036373732383739666166626631353738323830636234383430313539666433643139346232
-39626462613537363263343166663038346338383863343334656433396238346330653938373139
-39373265373961303763613962353762303735323966373762616462643433353162623136656130
-34666362373638303635313833663133623431353230386266653962646163646665333334646661
-38663634346461663430623235623138666563343739643432366635653331663233366136353666
-35393765346362663561633064356364623737353862353334386638306362363362386663663636
-37663731356664396330663862623965343237643066663031336338313937363461343264633534
-39613230373964663462656664656161623939656361336532383530303030666264323439623231
-65643430643433643239333338646363343933373934353435646337333238663239613539333862
-33333130386239303738366262386562323261653334356238646132313861396661633937353630
-65643064383036623035313766346636636264653265666232343837343033646466623066663932
-37313366363064613662383938663965633865326333323264363730666635316533393331643362
-64373661303761633862613237363430346134346237633736326139363963656239393162616432
-31663437333434626536353164663234353164623265623538303031333833646438616265306633
-38346535366531323032313232656533613431313133373561343465616266306634656237333038
-65633134656132623238363136623934346235316336326136613863626564396339646265323262
-65313239353737656231383664656431343239363730666362376532663835643337666664393738
-6131
+65333737373761626438343263333163623934626161313738303239383134333133313661333739
+3765666232653562383861643033356535383230613564330a343931393265303332346339373161
+31653534646463333138633564663238323664313432343666613633353538323530323631326665
+3133303461303966310a626664396637313532313666386236303765613530343863636636346334
+33633536656337643962663564656465666636623734376162366233643431343966373737613064
+62336665386635316433636166353263356131383632616665643935616131333230343965613834
+63323363616535363437306362613934633533386438353466353138386438313063316565616636
+66643535356364396230653032643661316534356266333035323766306165383562653836313532
+31396432316564633933363338393535363937386533343137373664366538323836343038313062
+61383630386233313034353966383265333735303064333535643738633362336362323565326131
+66666565376663383733616136386462353937613364653932353062386665623439613933366535
+61343134323031343133626265336231306131376661396163333939643561356363306333666637
+64353137643238653562643034383262356266366636333135616262643436363638666166336565
+63346131346238666166303338303264363634373635663830663636656661303935623239346339
+33306564313566343339626362333735343737333763616330303266353836303438323131306161
+61303633636335636335383734326638663238313961653561613164333865383364323234383133
+65626130376434343165373531643935616431316631636165323365376564646535613534616237
+65303430373336383436373162376536376563623730343237366435653163613337303538643062
+66643361613732366431336231363133326435623361663366646537386433613262326161303966
+39363732653361646534653866326436666462346235376664623039343431373938666266313034
+62373639323039656266623562326634633131623964313666646463383064303266643162636362
+35646563623533303466636631646339626464306665383266643839653734373465313538363035
+33313762313934396137323433313238393239623831663430396530303764336338356366646264
+36393038633033303066346339663939653964333735303465626139613464313437356264373562
+34376230333834373831363661636461383763383138653537383235343132623830326532393564
+61646265303835306534346433303138306632306163613336393834313337306233376665313262
+35396630666162373432313939646537666335343835613363653334313234356564373431366537
+37323838323835386538343261633762303035336665656638636165303130343733633766656333
+33623861633664626232316434326138303539363130333561323630393932363735363362663832
+64393965346131396236653864323930633763303435613330386236633164636465646664396530
+35343838323364643236383334663432316339613231613030643935333932633732313635633164
+63663861323663613931636238313862326364396538616463376533396136653266393136663265
+65613862333066643030656263333534343161613638356264663635643430356563313561633535
+30303165663931633761363633383237333765383332363962353530313036346561383539643966
+336562336464303538313234386162383165

roles/deploy-web/files/backup_canalblog.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+TAR=/usr/bin/tar
+PYTHON=/usr/bin/python3
+GZIP=/usr/bin/gzip
+SCRIPTDIR=/home/valentin/script
+WEBSCRAP=${SCRIPTDIR}/web_scrap.py
+URL=www.clarissariviere.com
+DATE=$(date +%Y%m%d)
+DIRECTORY=/home/valentin/backup
+BACKUPDIR=/home/valentin/backup_clarissa
+LIST=${BACKUPDIR}/backup.list
+fileBackup="backup-clarissa-${DATE}"
+LOGFILE=web_scrap.txt
+SENDER="valczebackup@gmail.com"
+if [ $(date +%u) -eq 1 ]; then
+	echo > ${LIST}
+	rm -rf "${BACKUPDIR}/*-incr.tar.gz"
+	fileBackup="${fileBackup}-full"
+	subject="Sauvegarde full"	
+else
+	fileBackup="${fileBackup}-incr"
+	subject="Sauvegarde incremental"
+fi
+subject="${subject} ${URL} ${DATE}"
+echo > ${BACKUPDIR}/${LOGFILE}
+${PYTHON} ${WEBSCRAP} --url ${URL} --dir ${DIRECTORY}  --logfile ${BACKUPDIR}/${LOGFILE} --quiet
+if [ ${?} -ne 0 ]; then
+	subject="FAIL : ${subject} : recuperation page"
+	echo ${subject} | mail -s "${subject}" -A ${BACKUPDIR}/${LOGFILE} ${SENDER}
+	exit 1
+fi
+${TAR} --create --file="${BACKUPDIR}/${fileBackup}.tar" --listed-incremental=${LIST} ${DIRECTORY}
+if [ ${?} -ne 0 ]; then
+	subject="FAIL : ${subject} : archivage page "
+	echo ${subject} | mail -s "${subject}" -A ${BACKUPDIR}/${LOGFILE} ${SENDER}
+	exit 1
+fi
+${GZIP} -f -9 "${BACKUPDIR}/${fileBackup}.tar"
+if [ ${?} -ne 0 ]; then
+	subject="FAIL :  ${subject} : compression archive "
+	echo ${subject} | mail -s "${subject}" -A ${BACKUPDIR}/${LOGFILE} ${SENDER}
+	exit 1
+fi
+subject="OK : ${subject}"
+echo ${subject}| mail -s "${subject}" -A ${BACKUPDIR}/${LOGFILE} ${SENDER}
+find ${BACKUPDIR} -name "*.tar.gz" -type f -ctime +90 -exec rm {} \;
+exit 0

roles/deploy-web/files/blacklist.sh

@@ -3,40 +3,62 @@ MAIL=/tmp/mail
 SERVER_LOG=/var/log/nginx
 HOST=($(cat /etc/sentinel/virtualhost))
 BLACKLIST=/etc/sentinel/blacklist
-currently_blacklist=$(ps -ef | grep blacklist | grep bash |grep ${USER}  | wc -l)
-
-if [ ${currently_blacklist} -eq 2 ]; then
-
-	chain_count=$(iptables -L BLACKLIST -n | wc -l)
-	if [ ${chain_count} -eq 0 ]; then
-		bash /usr/local/bin/sentinel/refill_blacklist.sh
-	fi
-
-	for i in ${HOST[@]}
-	do
-		log_access=${SERVER_LOG}/${i}_access.log
-		tail -n 50 $log_access | awk -F "|" '{ if($2 == "400" || $2 == "404") print $0}' > /tmp/error_$i
-		cat /tmp/error_$i | awk -F "|" '{ if($2 == "404") print $1}' > /tmp/404_$i
-		cat /tmp/error_$i | awk -F "|" '{ if($2 == "400") print $1}' > /tmp/400_$i
-		cat /tmp/404_$i | sort | uniq -c | awk '{ if($1 >= 5) print $2}' > /tmp/blacklist_404
-		cat /tmp/400_$i |sort | uniq -c |awk '{ if($1 >= 5) print $2}' > /tmp/blacklist_400
-		count=$(cat /tmp/blacklist_404 /tmp/blacklist_400 |grep -f ${BLACKLIST} -v |sort |uniq |wc -l)
-		if [ ${count} -ne 0 ]; then
-			echo "Nouvelle IP blacklisté" > ${MAIL}
-			list_ip=($(cat /tmp/blacklist_400 /tmp/blacklist_404 |grep -f ${BLACKLIST} -v  |sort |uniq))
-			for j in ${list_ip[@]}
-			do
-				echo ${j} >> ${MAIL}
-				curl http://ipinfo.io/${j} >> ${MAIL}
-				echo "" >> ${MAIL}
-				cat /tmp/error_$i | grep ${j} >> ${MAIL} 
-				echo "" >> ${MAIL}
-				echo ${j} >> ${BLACKLIST}
-				iptables -A BLACKLIST -s ${j} -j DROP
-			done
-			echo "IP dejà blacklisté : " >> ${MAIL}
-			cat ${BLACKLIST} >> ${MAIL}
-			cat ${MAIL} |mail -s "Blacklist IP ${i}" valczebackup@gmail.com
-		fi
-	done
+EXCLUDE=/etc/sentinel/exclude
+SENDER=/etc/sentinel/ip
+SSH=$(cat /etc/sentinel/ssh_port)
+IP=$(hostname -I |awk '{print $1}')
+chain_count=$(/usr/sbin/iptables -L BLACKLIST -n | wc -l)
+if [ ${chain_count} -eq 0 ]; then
+	bash /usr/local/bin/sentinel/refill_blacklist.sh
 fi
+
+list_sender=($(cat ${SENDER}))
+
+for i in ${list_sender[@]}
+do
+	if [ -f /tmp/blacklist_${i} ]; then
+		count_ip=$(cat ${BLACKLIST} /tmp/blacklist_${i} |grep -f ${EXCLUDE} -v |sort |uniq -ui |wc -l) 
+		cat ${BLACKLIST} /tmp/blacklist_${i} |grep -f ${EXCLUDE} -v |sort |uniq -u >> ${BLACKLIST}
+		if [ ${count_ip} -ne 0 ]; then
+			bash /usr/local/bin/sentinel/refill_blacklist.sh
+		fi
+	fi
+done
+
+for i in ${HOST[@]}
+do
+	log_access=${SERVER_LOG}/${i}_access.log
+	tail -n 50 $log_access | awk -F "|" '{ if($2 == "400" || $2 == "404") print $0}' > /tmp/error_$i
+	cat /tmp/error_$i | awk -F "|" '{ if($2 == "404") print $1}' > /tmp/404_$i
+	cat /tmp/error_$i | awk -F "|" '{ if($2 == "400") print $1}' > /tmp/400_$i
+	cat /tmp/404_$i | sort | uniq -c | awk '{ if($1 >= 5) print $2}' > /tmp/blacklist_404
+	cat /tmp/400_$i |sort | uniq -c |awk '{ if($1 >= 5) print $2}' > /tmp/blacklist_400
+	count=$(cat /tmp/blacklist_404 /tmp/blacklist_400 |grep -f ${BLACKLIST} -v |grep -f ${EXCLUDE} -v |sort |uniq |wc -l)
+	if [ ${count} -ne 0 ]; then
+		echo "Nouvelle IP blacklisté" > ${MAIL}
+		list_ip=($(cat /tmp/blacklist_400 /tmp/blacklist_404 |grep -f ${BLACKLIST} -v  |grep -f ${EXCLUDE} -v |sort |uniq))
+		for j in ${list_ip[@]}
+		do
+			echo ${j} >> ${MAIL}
+			curl http://ipinfo.io/${j} >> ${MAIL}
+			echo "" >> ${MAIL}
+			cat /tmp/error_$i | grep ${j} >> ${MAIL} 
+			echo "" >> ${MAIL}
+			echo ${j} >> ${BLACKLIST}
+			/usr/sbin/iptables -A BLACKLIST -s ${j} -j DROP
+		done
+		for j in ${list_sender[@]}
+		do
+			echo "Blacklist envoye a ${j}" >> ${MAIL}
+			echo "scp -i /home/valentin/.ssh-blacklist/id_rsa -P ${SSH} ${BLACKLIST} blacklist_user@${j}:/tmp/blacklist_${IP}" >> ${MAIL}
+			scp -i /home/valentin/.ssh-blacklist/id_rsa -P ${SSH} ${BLACKLIST} blacklist_user@${j}:/tmp/blacklist_${IP}i
+			if [ ${?} -ne 0 ]; then
+				echo "Error SCP for ${j}" >> ${MAIL}
+			fi
+		done
+		echo "IP dejà blacklisté : " >> ${MAIL}
+		cat ${BLACKLIST} >> ${MAIL}
+		cat ${MAIL} |mail -s "Blacklist IP ${i}" valczebackup@gmail.com
+	fi
+done
+

roles/deploy-web/files/gouter

@@ -1,11 +1,30 @@
 server {
+	if ($host = clarissariviere.fr) {
+        	return 301 "https://www.clarissariviere.com$request_uri";
+    	} # managed by Certbot
+	if ($host = www.clarissariviere.fr) {
+        	return 301 "https://www.clarissariviere.com$request_uri";
+    	} # managed by Certbot
+
+
+    	if ($host = clarissariviere.com) {
+        	return 301 "https://www.clarissariviere.com$request_uri";
+    	} # managed by Certbot
 	access_log /var/log/nginx/clarissa_access.log main;
 	error_log /var/log/nginx/clarissa_error.log;
 	#gzip_static off;
 	server_name clarissariviere.com clarissariviere.fr www.clarissariviere.fr www.clarissariviere.com;
         add_header 'Content-Security-Policy' 'upgrade-insecure-requests';
-	add_header Link "<https://www.clarissariviere.com; rel=\"canonical\">";
-        proxy_cache STATIC;	
+	#add_header Link "<https://www.clarissariviere.com$request_uri;> rel=\"canonical\", <https://www.clarissariviere.fr$request_uri;> rel=\"alternate\" hreflang=\"fr\"";
+	proxy_cache STATIC;	
+	set $header "<https://$host$request_uri;> rel=\"canonical\"";
+	if ($request_uri ~ "/tag/") {
+		set $header "<https://$host;> rel=\"canonical\"";
+		set $link "";
+
+	}
+	add_header Link $header;
+
 	location / {
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
@@ -26,8 +45,9 @@ server {
 		#proxy_set_header X-Real-IP $remote_addr;
                 #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		#proxy_set_header X-Forwarded-Proto $scheme;
-		sub_filter 'gouters.canalblog.com' "$host";
-		sub_filter '<meta name="generator" content="CanalBlog - https://www.canalblog.com" /> '';
+		sub_filter 'http://gouters.canalblog.com' "https://$host";
+		sub_filter '<meta name="generator" content="CanalBlog - https://www.canalblog.com" />' '';
+		sub_filter '<meta name="Keywords" content="' '<meta name="Keywords" content="clarissa, riviere, ';
 		sub_filter_types text/html text/xml text/plain text/css;
 		sub_filter_once off;	
 	
@@ -67,12 +87,12 @@ server {
 
 
     if ($host = clarissariviere.fr) {
-        return 301 https://$host$request_uri;
+        return 301 https://"www.clarissariviere.fr"$request_uri;
     } # managed by Certbot
 
 
     if ($host = clarissariviere.com) {
-        return 301 https://$host$request_uri;
+        return 301 https://"www.clarissariviere.com"$request_uri;
     } # managed by Certbot
 
 

roles/deploy-web/files/refill_blacklist.sh

@@ -1,11 +1,12 @@
 #!/bin/bash
 
+IPTABLES=/usr/sbin/iptables
 BLACKLIST=/etc/sentinel/blacklist
-chain_count=$(iptables -L BLACKLIST -n | wc -l)
+chain_count=$(${IPTABLES} -L BLACKLIST -n | wc -l)
 if [ ${chain_count} -eq 0 ]; then
-	iptables -N BLACKLIST
-	iptables -A INPUT -p tcp -m tcp --dport 80 -j BLACKLIST
-	iptables -A INPUT -p tcp -m tcp --dport 443 -j BLACKLIST
+	${IPTABLES} -N BLACKLIST
+	${IPTABLES} -I INPUT 1 -p tcp -m tcp --dport 80 -j BLACKLIST
+	${IPTABLES} -I INPUT 1 -p tcp -m tcp --dport 443 -j BLACKLIST
 
 fi
 if [ ! -f ${BLACKLIST} ]; then
@@ -13,7 +14,7 @@ if [ ! -f ${BLACKLIST} ]; then
 fi
 if [ ${chain_count} -gt 2 ]; then
 	chain_count=$(echo ${chain_count}-2 |bc)
-	iptables_ip=($(iptables -nvL BLACKLIST | tail -n ${chain_count}  | awk '{print $8}'))
+	iptables_ip=($(${IPTABLES} -nvL BLACKLIST | tail -n ${chain_count}  | awk '{print $8}'))
 	for i in $(cat ${BLACKLIST})
 	do
 		block_ip=1
@@ -24,13 +25,13 @@ if [ ${chain_count} -gt 2 ]; then
 			fi
 		done
 		if [ ${block_ip} -eq 1 ]; then
-			iptables -A BLACKLIST -s ${i} -j DROP
+			${IPTABLES} -A BLACKLIST -s ${i} -j DROP
 		fi
 	done
 else
 	for i in $(cat ${BLACKLIST})
 	do
-		iptables -A BLACKLIST -s ${i} -j DROP
+		${IPTABLES} -A BLACKLIST -s ${i} -j DROP
 	done
 fi
 

roles/deploy-web/files/scw-backup.sh

@@ -10,14 +10,12 @@ log () {
 
 rotate_log() {
 	cat ${LOGFILE_RECENT} >> ${LOGFILE}
-	backupScw=`echo ${SCW_BUCKET} | rev | cut -d "/" -f 2 | rev`
 	status="OK"
 	if [ $(grep "Errors 0" ${LOGFILE_RECENT} |wc -l) -eq 0 ]; then
 		status="ALERTE FAIL !!!"
 	fi
-	cat ${LOGFILE_RECENT} |mail -s "${status} | Backup ${backupScw} `date +%Y-%m-%d`" valczebackup@gmail.com
+	cat ${LOGFILE_RECENT} |mail -s "${status} | Backup ${SCW_BUCKET} `date +%Y-%m-%d`" valczebackup@gmail.com 
 }
-
 USER=$(whoami)
 currently_backuping=$(ps -ef | grep duplicity  | grep python |grep ${USER}  | wc -l)
 

roles/deploy-web/tasks/configure.yml

@@ -15,6 +15,11 @@
     src: virtualhost.j2
     dest: /etc/sentinel/virtualhost
     mode: "0444"
+  with_items:
+    - virtualhost
+    - ip
+    - ssh_port
+    - exclude
   vars:
     ansible_become: yes
     ansible_become_method: sudo

roles/deploy-web/templates/exclude.j2

@@ -0,0 +1,3 @@
+{% for host in ip_exclude_blacklist %}
+{{ host }}
+{% endfor %}

roles/deploy-web/templates/ip.j2

@@ -0,0 +1,3 @@
+{% for host in ip_sender_blacklist %}
+{{ host }}
+{% endfor %}

roles/deploy-web/templates/ssh_port.j2

@@ -0,0 +1 @@
+{{ ssh_port_blacklist }}

roles/deploy-web/templates/virtualhost.j2

@@ -1,3 +1,3 @@
 {% for host in virtualhosts %}
-    {{ host }}
+{{ host }}
 {% endfor %}