From 78d9357c020f54a080aa88c6c94c27b23e5dcfbe Mon Sep 17 00:00:00 2001
From: Gitea <gitea@git.nordlichter-brv.de>
Date: Sun, 29 Nov 2020 11:31:21 +0100
Subject: [PATCH] Rewritten to use /sys/block/$md/md/

---
 README.md                                     |   9 +-
 check_mdstat.sh                               | 268 +++++++-----------
 test/1missing.input                           |   4 -
 test/1missing.output                          |   1 -
 test/OK1.input                                |   6 -
 test/OK1.output                               |   1 -
 test/OK2.input                                |   6 -
 test/OK2.output                               |   1 -
 test/OK3.input                                |  14 -
 test/OK3.output                               |   1 -
 test/bitmap.input                             |   7 -
 test/bitmap.output                            |   1 -
 test/notExisting.input                        |   5 -
 test/notExisting.output                       |   2 +-
 test/recovering.input/md0/md/array_size       |   1 +
 test/recovering.input/md0/md/array_state      |   1 +
 test/recovering.input/md0/md/chunk_size       |   1 +
 test/recovering.input/md0/md/component_size   |   1 +
 test/recovering.input/md0/md/degraded         |   1 +
 test/recovering.input/md0/md/dev-sda1/slot    |   1 +
 test/recovering.input/md0/md/dev-sda1/state   |   1 +
 test/recovering.input/md0/md/dev-sdc1/slot    |   1 +
 test/recovering.input/md0/md/dev-sdc1/state   |   1 +
 test/recovering.input/md0/md/dev-sdd1/slot    |   1 +
 test/recovering.input/md0/md/dev-sdd1/state   |   1 +
 test/recovering.input/md0/md/last_sync_action |   1 +
 test/recovering.input/md0/md/level            |   1 +
 test/recovering.input/md0/md/max_read_errors  |   1 +
 test/recovering.input/md0/md/raid_disks       |   1 +
 test/recovering.input/md0/md/rd0              |   1 +
 test/recovering.input/md0/md/rd1              |   1 +
 test/recovering.input/md0/md/rd2              |   1 +
 test/recovering.input/md0/md/sync_action      |   1 +
 test/recovering.input/md0/md/sync_completed   |   1 +
 test/recovering.input/md0/md/sync_speed       |   1 +
 .../recovering.input/md0/queue/hw_sector_size |   1 +
 test/recovering.output                        |   1 +
 test/recovery.input                           |   6 -
 test/recovery.output                          |   1 -
 test/run.sh                                   |   5 +-
 40 files changed, 134 insertions(+), 227 deletions(-)
 delete mode 100644 test/1missing.input
 delete mode 100644 test/1missing.output
 delete mode 100644 test/OK1.input
 delete mode 100644 test/OK1.output
 delete mode 100644 test/OK2.input
 delete mode 100644 test/OK2.output
 delete mode 100644 test/OK3.input
 delete mode 100644 test/OK3.output
 delete mode 100644 test/bitmap.input
 delete mode 100644 test/bitmap.output
 delete mode 100644 test/notExisting.input
 create mode 100644 test/recovering.input/md0/md/array_size
 create mode 100644 test/recovering.input/md0/md/array_state
 create mode 100644 test/recovering.input/md0/md/chunk_size
 create mode 100644 test/recovering.input/md0/md/component_size
 create mode 100644 test/recovering.input/md0/md/degraded
 create mode 100644 test/recovering.input/md0/md/dev-sda1/slot
 create mode 100644 test/recovering.input/md0/md/dev-sda1/state
 create mode 100644 test/recovering.input/md0/md/dev-sdc1/slot
 create mode 100644 test/recovering.input/md0/md/dev-sdc1/state
 create mode 100644 test/recovering.input/md0/md/dev-sdd1/slot
 create mode 100644 test/recovering.input/md0/md/dev-sdd1/state
 create mode 100644 test/recovering.input/md0/md/last_sync_action
 create mode 100644 test/recovering.input/md0/md/level
 create mode 100644 test/recovering.input/md0/md/max_read_errors
 create mode 100644 test/recovering.input/md0/md/raid_disks
 create mode 120000 test/recovering.input/md0/md/rd0
 create mode 120000 test/recovering.input/md0/md/rd1
 create mode 120000 test/recovering.input/md0/md/rd2
 create mode 100644 test/recovering.input/md0/md/sync_action
 create mode 100644 test/recovering.input/md0/md/sync_completed
 create mode 100644 test/recovering.input/md0/md/sync_speed
 create mode 100644 test/recovering.input/md0/queue/hw_sector_size
 create mode 100644 test/recovering.output
 delete mode 100644 test/recovery.input
 delete mode 100644 test/recovery.output

diff --git a/README.md b/README.md
index dfd7046..daed693 100644
--- a/README.md
+++ b/README.md
@@ -1,18 +1,17 @@
 # check_mdstat.sh
-This script reads from `/proc/mdstat` to get the status of a RAID devices.
+This script reads from `/sys/block/$md/md/` to get the status of a RAID devices.
 
-Other than [check_md_raid](https://exchange.icinga.com/exchange/check_md_raid) and [check_md_raid updated](https://exchange.icinga.com/jasem/check_md_raid updated/) not all RAIDs are checked simulataneously, but only one specific. This gives the possibility to query some "performance" data and later add more granular checks.
+Other than [check_md_raid](https://exchange.icinga.com/exchange/check_md_raid) and [check_md_raid updated](https://exchange.icinga.com/jasem/check_md_raid updated/) the data is not taken from /proc/mdstat which is less machine readable. Also not all RAIDs are checked simulataneously but only one specific. This gives the possibility to query some "performance" data and later add more granular checks.
 
 ## Arguments
 * --raid | -r: Name of the RAID, e.g. `md0`
-* --input | -i: Input file. Defaults to /proc/mdstat
+* --input | -i: Input base directory. Defaults to /sys/block/
 * --debug | -d: More debugging output, cannot be used with icinga2
 
 ## Results
 * CRITICAL: if any device failed
 * CRITICAL: if the state of the RAID is not active/started
+* CRITICAL: if the number of active devices is smaller than the number of configured devices.
 * WARNING: when recoverying/resyncing
-* CRITICAL: if the number of active devices is smaller than the number of active devices.
 
 ## Known issues
-Probably I did not get things right with spare devices.
\ No newline at end of file
diff --git a/check_mdstat.sh b/check_mdstat.sh
index ba320f4..55c3d92 100755
--- a/check_mdstat.sh
+++ b/check_mdstat.sh
@@ -14,7 +14,7 @@ if [ $# -lt 1 -o -z "$1" ]; then
 	exit 1;
 fi
 
-INPUT="/proc/mdstat"
+INPUT="/sys/block"
 DEBUG=false
 while [[ $# -gt 0 ]]; do
 	case $1 in
@@ -35,133 +35,92 @@ if [ -z "$TARGET_RAID_NAME" ]; then
 	echo "ERROR: pass --raid as argument"
 	exit 255
 fi
+INPUT="$INPUT/$TARGET_RAID_NAME/md"
+
+if [ ! -d $INPUT ]; then
+	echo "CRITICAL: RAID $TARGET_RAID_NAME not found in $INPUT"
+	exit 2
+fi
 
 function log {
 	if $DEBUG; then
 		echo "  > $@"
 	fi
 }
-function logLine {
-	if $DEBUG; then
-		echo "  : $@"
-	fi
-}
 NL=$'\n'
+log "Reading from $INPUT"
 
-CURRENT_RAID=	# string
-function setCurrentRaid {
-	# This function is called for each line in $1
-	# If the line does not start with a whitespace, it denotes a new RAID device and CURRENT_RAID is set
-	# Also, the state and the devices are parsed 
-	local line="$1"
-	if [[ $line =~ ^[^\s].+:.+ ]]; then
-		local x1="${line%:*}" # Remove : suffix
-		local x2="${x1% *}"   # Trim right
-		CURRENT_RAID="$x2"
-		log "SET CURRENT RAID"
-	fi
-	log "CURRENT RAID: $CURRENT_RAID"
-}
+STATE=							# string
+LEVEL=							# int
+SIZE_IN_BLOCKS="!"				# int
+CHUNK_SIZE_IN_BYTES=			# size in bytes for chunks
+NUM_DEVICES=0					# int with total number of devices in raid
+NUM_ACTIVE_DEVICES=0			# int with number of active devices in raid
+NUM_SPARE_DEVICES=0				# int with number of spare devices in raid
+RECOVERY_PROGRESS=				# float in percent
+RECOVERY_SPEED=					# sync speed in K/s
+declare -A DEVICE_STATE			# dict of strings. key = dev, value = state
+declare -A FAULTY_DEVICES		# dict with only faulty devices. key = dev, value = true
+declare -A TO_REPLACE_DEVICES	# dict of devices in raid that shall be replaced. key = dev, value = true
 
-STATE=		# "active", "started", "inactive"
-LEVEL=		# int
-declare -A DEVICES	# dict of strings. key = index, value = dev name
-declare -A FAILED_DEVICES
-function setStateAndDevicesAndLevel {
-	# This functions is called for the line that starts a new RAID device by setCurrentRaid, with the complete line in $1
-	# It parses the STATE, the LEVEL and the DEVICES
-	local line="$1"
-	local x1="${line#*: }" 				# Remove : prefix
-	STATE="${x1%% *}" 					# Only the first word
-	local x2="${x1:$((${#STATE}+1))}"	# Remove the first word
-	LEVEL="${x2%% *}" 					# Only the first word
-	local x3="${x2:$((${#LEVEL}+1))}"	# Remove the first word
+function getDataFromSys {
+	# https://mjmwired.net/kernel/Documentation/md.txt
 
-	LEVEL="${LEVEL:4}"					# Remove the raid prefix
+	LEVEL=$(<$INPUT/level)
+	NUM_DEVICES=$(<$INPUT/raid_disks)
+	CHUNK_SIZE_IN_BYTES=$(<$INPUT/chunk_size)
+	SIZE_IN_BLOCKS=$(<$INPUT/component_size)
+	STATE=$(<$INPUT/array_state)
+	RECOVERY_PROGRESS=$(<$INPUT/sync_completed)
+	RECOVERY_SPEED=$(<$INPUT/sync_speed)
 
-	for dev1 in $x3; do
-		local devName="${dev1%[*}"		# device name
-		local dev2="${dev1#*[}"			# all after device name
-		local devIndex="${dev2%]*}"		# device index
-		local dev3="${dev2#*]}"			# all after device index
-		local devFailed=false
-		DEVICES[$devIndex]="$devName"
-		FAILED_DEVICES[$devIndex]=false
-		if [ -n "$dev3" ]; then
-			FAILED_DEVICES[$devIndex]=true
-		fi
+	log "LEVEL =               $LEVEL"
+	log "STATE =               $STATE"
+	log "CHUNK_SIZE_IN_BYTES = $CHUNK_SIZE_IN_BYTES"
+	log "SIZE_IN_BLOCKS =      $SIZE_IN_BLOCKS"
+	log "RECOVERY_PROGRESS =   $RECOVERY_PROGRESS"
+	log "RECOVERY_SPEED =      $RECOVERY_SPEED"
+
+	for dev in `find $INPUT -type d -name 'dev-*' -printf "%f\n"`; do
+		local deviceName=${dev:4}
+		local slot=$(<$INPUT/$dev/slot)
+		local state=$(<$INPUT/$dev/state)
+		log "$dev [$slot] is '$state'"
+		DEVICE_STATE[$deviceName]=$state
+		case "$state" in
+			*in_sync*) 			((NUM_ACTIVE_DEVICES++));;
+			*writemostly*)		((NUM_ACTIVE_DEVICES++));;
+			*faulty*) 			FAULTY_DEVICES[$deviceName]=true;;
+			*blocked*)	 		FAULTY_DEVICES[$deviceName]=true;;
+			*write-error*)		TO_REPLACE_DEVICES[$deviceName]=true;;
+			*want_replacement*)	TO_REPLACE_DEVICES[$deviceName]=true;;
+			*replacement*)		((NUM_SPARE_DEVICES++));;
+			*spare*)			((NUM_SPARE_DEVICES++));;
+			*)			 		FAULTY_DEVICES[$deviceName]=true;;
+		esac
 	done
 
-	log "STATE = $STATE"
-	log "LEVEL = $LEVEL"
-	log "DEVICES = ${DEVICES[@]}"
-}
+	log "NUM_DEVICES =         $NUM_DEVICES"
+	log "NUM_ACTIVE_DEVICES =  $NUM_ACTIVE_DEVICES"
+	log "NUM_SPARE_DEVICES =   $NUM_SPARE_DEVICES"
+	log "FAULTY_DEVICES =      ${FAULTY_DEVICES[*]}"
+	log "TO_REPLACE_DEVICES =  ${TO_REPLACE_DEVICES[*]}"
 
-SIZE_IN_BLOCKS="!"			# int
-NUM_DEVICES=0				# int with total number of devices in raid
-NUM_ACTIVE_DEVICES=0		# int with number of active devices in raid
-function parseConfigStatusLine {
-	# This function is called for 1st line after the raid definition line
-	# It parses the SIZE_IN_BLOCKS
-	local line="$1"
-	line=$(echo $line)	# trim left
-	SIZE_IN_BLOCKS=${line%% *}
-
-	local x1=${line##* }		# Get the last word [UU_]
-	local x2=${line% *}			# Remove the last word
-	local x3=${x2##* }			# Get the last word [3/2]
-	x3=${x3:1:-1}
-	NUM_DEVICES=${x3%/*}
-	NUM_ACTIVE_DEVICES=${x3#*/}
-
-
-	log "SIZE_IN_BLOCKS = $SIZE_IN_BLOCKS"
-	log "NUM_DEVICES = $NUM_DEVICES"
-	log "NUM_ACTIVE_DEVICES = $NUM_ACTIVE_DEVICES"
-	# TODO
-}
-
-function parseBitmapOrRecoveryLine {
-	# This function is called for the 2nd line after the raid definition line
-	# It checks whether it contains a bitmap or a recovery line
-	if [[ "$1" == *bitmap* ]]; then
-		parseBitmapLine "$1"
+	if [ "$RECOVERY_PROGRESS" != "none" ]; then
+		RECOVERY_NUM=${RECOVERY_PROGRESS% /*}
+		log "'$RECOVERY_NUM'"
+		RECOVERY_DEN=${RECOVERY_PROGRESS#*/ }
+		log "'$RECOVERY_DEN'"
+		RECOVERY_PERCENT=$(($RECOVERY_NUM*100/$RECOVERY_DEN))
+		BLOCK_SIZE=$(<$(dirname $INPUT)/queue/hw_sector_size)
+		RECOVERY_REMAINING_S=$(( ($RECOVERY_DEN-$RECOVERY_NUM)*$BLOCK_SIZE/1024/$RECOVERY_SPEED))
+		RECOVERY_REMAINING_MIN=$(($RECOVERY_REMAINING_S/60))
 	fi
-	if [[ "$1" == *recovery* ]]; then
-		parseRecoveryLine "$1"
-	fi
-}
-
-function parseBitmapLine {
-	local x
-	# TODO
-}
-
-RECOVERY_PROGRESS=	# float in percent
-RECOVERY_INFO=		# finish and speed
-function parseRecoveryLine {
-	# This function is called for the 2nd or 3rd lineafter the raid definition line
-	# It sets the RECOVERY_PROGRESS
-	local line="$1"
-	
-	local x1="${line##*recovery = }"
-	local x2="${x1%%%*}"
-	RECOVERY_PROGRESS="$x2"
-	
-	local x3="${line##*finish=}"
-	RECOVERY_INFO="finish=$x3"
-
-	log "RECOVERY_PROGESS = $RECOVERY_PROGRESS"
-	log "RECOVERY_INFO = $RECOVERY_INFO"
 }
 
 function printOutputAndExit {
-	# STATE, LEVEL, DEVICES, SIZE_IN_BLOCKS, NUM_DEVICES, BAD_DEVICES, RECOVERY_PROGRESS, RECOVERY_INFO
-
-	local result="OK"
+	local result=""
 	local info=""
-	local numFailedDevices=0
-	local numSpareDevices=0
 
 	if [ $NUM_ACTIVE_DEVICES -lt $NUM_DEVICES ]; then
 		result="CRITICAL"
@@ -169,56 +128,54 @@ function printOutputAndExit {
 		info="${info}Missing $(($NUM_DEVICES - $NUM_ACTIVE_DEVICES)) of $NUM_DEVICES devices"
 	fi
 
-	if [ -n "$RECOVERY_PROGRESS" ]; then
-		result="WARNING"
-		if [ -n "$info" ]; then info="$info. "; fi
-		info="${info}Recovering: progress=$RECOVERY_PROGRESS% $RECOVERY_INFO"
-	fi
-
-	local devIndices="$(printf "%s\n" "${!DEVICES[@]}" | sort -n)"
-	for devIndex in $devIndices; do
-		if [ -z "${DEVICES[$devIndex]}" ]; then
-			((numSpareDevies++))
-			continue
-		fi
-		if ${FAILED_DEVICES[$devIndex]}; then
-			result="CRITICAL"
-			((numFailedDevices++))
-			if [ -n "$info" ]; then info="$info. "; fi
-			info="${info}Device ${DEVICES[$devIndex]} failed"
-		fi
-	done
-
-	if [ "$STATE" != "active" -a "$STATE" != "started" ]; then
+	if [ ${#FAULTY_DEVICES[@]} -gt 0 ]; then
 		result="CRITICAL"
 		if [ -n "$info" ]; then info="$info. "; fi
-		info="${info}State is $STATE"
+		info="${info}These devices failed: ${!FAULTY_DEVICES[@]}"
 	fi
 
+	if [ ${#TO_REPLACE_DEVICES[@]} -gt 0 ]; then
+		if [ -z "$result" ]; then result="WARNING"; fi
+		if [ -n "$info" ]; then info="$info. "; fi
+		info="${info}These devices should be replaced: ${!TO_REPLACE_DEVICES[@]}"
+	fi
+
+	if [ "$RECOVERY_PROGRESS" != "none" ]; then
+		if [ -z "$result" ]; then result="WARNING"; fi
+		if [ -n "$info" ]; then info="$info. "; fi
+		info="${info}Recovering: $RECOVERY_PERCENT%, remaining ${RECOVERY_REMAINING_MIN}min"
+	fi
+
+	if [ -z "$result" ]; then result="OK"; fi
+
 	echo -n "$result"
 	if [ -n "$info" ]; then
 		echo -n ": $info"
 	fi
 
 	echo -n " |"
-	echo -n " 'raid level'=$LEVEL"
+	echo -n " 'raid level: $LEVEL'=0"
 	echo -n " 'size in blocks'=$SIZE_IN_BLOCKS"
 	echo -n " 'num devices'=$NUM_DEVICES"
-	echo -n " 'num failed devices'=$numFailedDevices;;1;0;$NUM_DEVICES"
-	echo -n " 'num spare devices'=$numSpareDevices;;;0;$NUM_DEVICES"
-	if [ -n "$RECOVERY_PROGRESS" ]; then
-		echo -n " 'recovery progress'=$RECOVERY_PROGRESS%;0;;0;100"
+	echo -n " 'num active devices'=$NUM_ACTIVE_DEVICES;;;0;$NUM_DEVICES"
+	echo -n " 'num failed devices'=${#FAILED_DEVICES[@]};;1;0;$NUM_DEVICES"
+	echo -n " 'num spare devices'=$NUM_SPARE_DEVICES;;;0"
+	if [ "$RECOVERY_PROGRESS" != "none" ]; then
+		if [ -z "$result" ]; then result="WARNING"; fi
+		if [ -n "$info" ]; then info="$info. "; fi
+		echo -n " 'recovery progress'=$RECOVERY_NUM;0;;0;$RECOVERY_DEN"
+		echo -n " 'recovery speed [1/s]'=${RECOVERY_SPEED}K"
+		echo -n " 'recovery remaining'=${RECOVERY_REMAINING_S}s"
 	fi
-	for devIndex in $devIndices; do
-		if [ -z "${DEVICES[$devIndex]}" ]; then
-			echo -n " 'device $devIndex: spare'=$devIndex"
-			continue
+	for dev in ${!DEVICE_STATE[@]}; do
+		echo -n " 'dev $dev: ${DEVICE_STATE[$dev]}"
+		if [ -n "${FAULTY_DEVICE[$dev]}" ]; then
+			echo -n " faulty"
 		fi
-		if ${FAILED_DEVICES[$devIndex]}; then
-			echo -n " 'device $devIndex: ${DEVICES[$devIndex]} failed'=$devIndex;;$devIndex'"
-			continue
+		if [ -n "${TO_REPLACE_DEVICE[$dev]}" ]; then
+			echo -n " wants replacement"
 		fi
-		echo -n " 'device $devIndex: ${DEVICES[$devIndex]}'=$devIndex"
+		echo -n "'=0"
 	done
 
 	echo
@@ -231,25 +188,6 @@ function printOutputAndExit {
 }
 
 
-HAD_TARGET_RAID=0
-
-while IFS= read -r line; do
-    logLine "$line"
-
-	setCurrentRaid "$line"
-
-	if [ "$CURRENT_RAID" == "$TARGET_RAID_NAME" ]; then
-		case $HAD_TARGET_RAID in
-			0) setStateAndDevicesAndLevel "$line";;
-			1) parseConfigStatusLine "$line";;
-			*) parseBitmapOrRecoveryLine "$line";;
-		esac
-		((HAD_TARGET_RAID++))
-	elif [ $HAD_TARGET_RAID -gt 0 ]; then
-		printOutputAndExit
-	fi
-done < $INPUT
-
-echo "CRITICAL: RAID $TARGET_RAID_NAME not found in $(basename $INPUT)"
-exit 2
+getDataFromSys
+printOutputAndExit
 
diff --git a/test/1missing.input b/test/1missing.input
deleted file mode 100644
index 97ad269..0000000
--- a/test/1missing.input
+++ /dev/null
@@ -1,4 +0,0 @@
-Personalities : [raid6] [raid5] [raid4]
-md0 : active raid5 sda1[0] sdd1[2] sdb1[1]
-     1465151808 blocks level 5, 64k chunk, algorithm 2 [4/3] [UUU_]
-    unused devices: <none>
diff --git a/test/1missing.output b/test/1missing.output
deleted file mode 100644
index 282cb3a..0000000
--- a/test/1missing.output
+++ /dev/null
@@ -1 +0,0 @@
-CRITICAL: Missing 1 of 4 devices | 'raid level'=5 'size in blocks'=1465151808 'num devices'=4 'num failed devices'=0;;1;0;4 'num spare devices'=0;;;0;4 'device 0: sda1'=0 'device 1: sdb1'=1 'device 2: sdd1'=2
diff --git a/test/OK1.input b/test/OK1.input
deleted file mode 100644
index bae8d9a..0000000
--- a/test/OK1.input
+++ /dev/null
@@ -1,6 +0,0 @@
-Personalities : [raid1] 
-md0 : active raid1 sdb1[1] sda1[0]
-      976430080 blocks super 1.2 [2/2] [UU]
-      bitmap: 0/8 pages [0KB], 65536KB chunk
-
-unused devices: <none>
diff --git a/test/OK1.output b/test/OK1.output
deleted file mode 100644
index 8c89504..0000000
--- a/test/OK1.output
+++ /dev/null
@@ -1 +0,0 @@
-OK | 'raid level'=1 'size in blocks'=976430080 'num devices'=2 'num failed devices'=0;;1;0;2 'num spare devices'=0;;;0;2 'device 0: sda1'=0 'device 1: sdb1'=1
diff --git a/test/OK2.input b/test/OK2.input
deleted file mode 100644
index 7a6f761..0000000
--- a/test/OK2.input
+++ /dev/null
@@ -1,6 +0,0 @@
-Personalities : [raid1] [raid6] [raid5] [raid4]
-md0 : active raid5 sde1[0] sdf1[4] sdb1[5] sdd1[2] sdc1[1]
-      1250241792 blocks super 1.2 level 5, 64k chunk, algorithm 2 [5/5] [UUUUU]
-      bitmap: 0/10 pages [0KB], 16384KB chunk
-
-unused devices: <none>
diff --git a/test/OK2.output b/test/OK2.output
deleted file mode 100644
index f033258..0000000
--- a/test/OK2.output
+++ /dev/null
@@ -1 +0,0 @@
-OK | 'raid level'=5 'size in blocks'=1250241792 'num devices'=5 'num failed devices'=0;;1;0;5 'num spare devices'=0;;;0;5 'device 0: sde1'=0 'device 1: sdc1'=1 'device 2: sdd1'=2 'device 4: sdf1'=4 'device 5: sdb1'=5
diff --git a/test/OK3.input b/test/OK3.input
deleted file mode 100644
index f338ed7..0000000
--- a/test/OK3.input
+++ /dev/null
@@ -1,14 +0,0 @@
-Personalities : [raid1] [raid6] [raid5] [raid4]
-md1 : active raid1 sdb2[1] sda2[0]
-      136448 blocks [2/2] [UU]
-
-md2 : active raid1 sdb3[1] sda3[0]
-      129596288 blocks [2/2] [UU]
-
-md0 : active raid5 sdl1[9] sdk1[8] sdj1[7] sdi1[6] sdh1[5] sdg1[4] sdf1[3] sde1[2] sdd1[1] sdc1[0]
-      1318680576 blocks level 5, 1024k chunk, algorithm 2 [10/10] [UUUUUUUUUU]
-
-md3 : active raid1 sdb1[1] sda1[0]
-      16787776 blocks [2/2] [UU]
-
-unused devices: <none> 
diff --git a/test/OK3.output b/test/OK3.output
deleted file mode 100644
index 8607af1..0000000
--- a/test/OK3.output
+++ /dev/null
@@ -1 +0,0 @@
-OK | 'raid level'=5 'size in blocks'=1318680576 'num devices'=10 'num failed devices'=0;;1;0;10 'num spare devices'=0;;;0;10 'device 0: sdc1'=0 'device 1: sdd1'=1 'device 2: sde1'=2 'device 3: sdf1'=3 'device 4: sdg1'=4 'device 5: sdh1'=5 'device 6: sdi1'=6 'device 7: sdj1'=7 'device 8: sdk1'=8 'device 9: sdl1'=9
diff --git a/test/bitmap.input b/test/bitmap.input
deleted file mode 100644
index fdccafc..0000000
--- a/test/bitmap.input
+++ /dev/null
@@ -1,7 +0,0 @@
-Personalities : [linear] [raid0] [raid1] [raid5] [raid4] [raid6]
-md0 : active raid6 sdf1[0] sde1[1] sdd1[2] sdc1[3] sdb1[4] sda1[5] hdb1[6]
-      1225557760 blocks level 6, 256k chunk, algorithm 2 [7/7] [UUUUUUU]
-      bitmap: 0/234 pages [0KB], 512KB chunk
-
-unused devices: <none>
-
diff --git a/test/bitmap.output b/test/bitmap.output
deleted file mode 100644
index 83c86f1..0000000
--- a/test/bitmap.output
+++ /dev/null
@@ -1 +0,0 @@
-OK | 'raid level'=6 'size in blocks'=1225557760 'num devices'=7 'num failed devices'=0;;1;0;7 'num spare devices'=0;;;0;7 'device 0: sdf1'=0 'device 1: sde1'=1 'device 2: sdd1'=2 'device 3: sdc1'=3 'device 4: sdb1'=4 'device 5: sda1'=5 'device 6: hdb1'=6
diff --git a/test/notExisting.input b/test/notExisting.input
deleted file mode 100644
index a3df9a2..0000000
--- a/test/notExisting.input
+++ /dev/null
@@ -1,5 +0,0 @@
-Personalities : [raid1] [raid6] [raid5] [raid4]
-md1 : active raid1 sdb2[1] sda2[0]
-      136448 blocks [2/2] [UU]
-
-unused devices: <none> 
diff --git a/test/notExisting.output b/test/notExisting.output
index b5ca907..0055521 100644
--- a/test/notExisting.output
+++ b/test/notExisting.output
@@ -1 +1 @@
-CRITICAL: RAID md0 not found in notExisting.input
+CRITICAL: RAID md0 not found in ./notExisting.input/md0/md
diff --git a/test/recovering.input/md0/md/array_size b/test/recovering.input/md0/md/array_size
new file mode 100644
index 0000000..4ad96d5
--- /dev/null
+++ b/test/recovering.input/md0/md/array_size
@@ -0,0 +1 @@
+default
diff --git a/test/recovering.input/md0/md/array_state b/test/recovering.input/md0/md/array_state
new file mode 100644
index 0000000..8312630
--- /dev/null
+++ b/test/recovering.input/md0/md/array_state
@@ -0,0 +1 @@
+clean
diff --git a/test/recovering.input/md0/md/chunk_size b/test/recovering.input/md0/md/chunk_size
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/test/recovering.input/md0/md/chunk_size
@@ -0,0 +1 @@
+0
diff --git a/test/recovering.input/md0/md/component_size b/test/recovering.input/md0/md/component_size
new file mode 100644
index 0000000..6086cfe
--- /dev/null
+++ b/test/recovering.input/md0/md/component_size
@@ -0,0 +1 @@
+976431104
diff --git a/test/recovering.input/md0/md/degraded b/test/recovering.input/md0/md/degraded
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/test/recovering.input/md0/md/degraded
@@ -0,0 +1 @@
+1
diff --git a/test/recovering.input/md0/md/dev-sda1/slot b/test/recovering.input/md0/md/dev-sda1/slot
new file mode 100644
index 0000000..0cfbf08
--- /dev/null
+++ b/test/recovering.input/md0/md/dev-sda1/slot
@@ -0,0 +1 @@
+2
diff --git a/test/recovering.input/md0/md/dev-sda1/state b/test/recovering.input/md0/md/dev-sda1/state
new file mode 100644
index 0000000..9649910
--- /dev/null
+++ b/test/recovering.input/md0/md/dev-sda1/state
@@ -0,0 +1 @@
+in_sync
diff --git a/test/recovering.input/md0/md/dev-sdc1/slot b/test/recovering.input/md0/md/dev-sdc1/slot
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/test/recovering.input/md0/md/dev-sdc1/slot
@@ -0,0 +1 @@
+1
diff --git a/test/recovering.input/md0/md/dev-sdc1/state b/test/recovering.input/md0/md/dev-sdc1/state
new file mode 100644
index 0000000..9649910
--- /dev/null
+++ b/test/recovering.input/md0/md/dev-sdc1/state
@@ -0,0 +1 @@
+in_sync
diff --git a/test/recovering.input/md0/md/dev-sdd1/slot b/test/recovering.input/md0/md/dev-sdd1/slot
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/test/recovering.input/md0/md/dev-sdd1/slot
@@ -0,0 +1 @@
+0
diff --git a/test/recovering.input/md0/md/dev-sdd1/state b/test/recovering.input/md0/md/dev-sdd1/state
new file mode 100644
index 0000000..cb75ec0
--- /dev/null
+++ b/test/recovering.input/md0/md/dev-sdd1/state
@@ -0,0 +1 @@
+spare
diff --git a/test/recovering.input/md0/md/last_sync_action b/test/recovering.input/md0/md/last_sync_action
new file mode 100644
index 0000000..2894e25
--- /dev/null
+++ b/test/recovering.input/md0/md/last_sync_action
@@ -0,0 +1 @@
+recovery
diff --git a/test/recovering.input/md0/md/level b/test/recovering.input/md0/md/level
new file mode 100644
index 0000000..341b806
--- /dev/null
+++ b/test/recovering.input/md0/md/level
@@ -0,0 +1 @@
+raid1
diff --git a/test/recovering.input/md0/md/max_read_errors b/test/recovering.input/md0/md/max_read_errors
new file mode 100644
index 0000000..209e3ef
--- /dev/null
+++ b/test/recovering.input/md0/md/max_read_errors
@@ -0,0 +1 @@
+20
diff --git a/test/recovering.input/md0/md/raid_disks b/test/recovering.input/md0/md/raid_disks
new file mode 100644
index 0000000..00750ed
--- /dev/null
+++ b/test/recovering.input/md0/md/raid_disks
@@ -0,0 +1 @@
+3
diff --git a/test/recovering.input/md0/md/rd0 b/test/recovering.input/md0/md/rd0
new file mode 120000
index 0000000..4a08007
--- /dev/null
+++ b/test/recovering.input/md0/md/rd0
@@ -0,0 +1 @@
+dev-sdd1
\ No newline at end of file
diff --git a/test/recovering.input/md0/md/rd1 b/test/recovering.input/md0/md/rd1
new file mode 120000
index 0000000..d3a806e
--- /dev/null
+++ b/test/recovering.input/md0/md/rd1
@@ -0,0 +1 @@
+dev-sdc1
\ No newline at end of file
diff --git a/test/recovering.input/md0/md/rd2 b/test/recovering.input/md0/md/rd2
new file mode 120000
index 0000000..d5e8cf1
--- /dev/null
+++ b/test/recovering.input/md0/md/rd2
@@ -0,0 +1 @@
+dev-sda1
\ No newline at end of file
diff --git a/test/recovering.input/md0/md/sync_action b/test/recovering.input/md0/md/sync_action
new file mode 100644
index 0000000..f3880fa
--- /dev/null
+++ b/test/recovering.input/md0/md/sync_action
@@ -0,0 +1 @@
+recover
diff --git a/test/recovering.input/md0/md/sync_completed b/test/recovering.input/md0/md/sync_completed
new file mode 100644
index 0000000..50560f9
--- /dev/null
+++ b/test/recovering.input/md0/md/sync_completed
@@ -0,0 +1 @@
+1728697856 / 1952862208
diff --git a/test/recovering.input/md0/md/sync_speed b/test/recovering.input/md0/md/sync_speed
new file mode 100644
index 0000000..79a764c
--- /dev/null
+++ b/test/recovering.input/md0/md/sync_speed
@@ -0,0 +1 @@
+17170
diff --git a/test/recovering.input/md0/queue/hw_sector_size b/test/recovering.input/md0/queue/hw_sector_size
new file mode 100644
index 0000000..4d0e90c
--- /dev/null
+++ b/test/recovering.input/md0/queue/hw_sector_size
@@ -0,0 +1 @@
+512
diff --git a/test/recovering.output b/test/recovering.output
new file mode 100644
index 0000000..713d2b4
--- /dev/null
+++ b/test/recovering.output
@@ -0,0 +1 @@
+CRITICAL: Missing 1 of 3 devices. Recovering: 88%, remaining 108min | 'raid level: raid1'=0 'size in blocks'=976431104 'num devices'=3 'num active devices'=2;;;0;3 'num failed devices'=0;;1;0;3 'num spare devices'=1;;;0 'recovery progress'=1728697856;0;;0;1952862208 'recovery speed [1/s]'=17170K 'recovery remaining'=6527s 'dev sdd1: spare'=0 'dev sdc1: in_sync'=0 'dev sda1: in_sync'=0
diff --git a/test/recovery.input b/test/recovery.input
deleted file mode 100644
index 6e8ddc6..0000000
--- a/test/recovery.input
+++ /dev/null
@@ -1,6 +0,0 @@
-Personalities : [raid1] [raid6] [raid5] [raid4]
-md0 : active raid5 sdh1[6] sdg1[4] sdf1[3] sde1[2] sdd1[1] sdc1[0]
-      1464725760 blocks level 5, 64k chunk, algorithm 2 [6/5] [UUUUUU]
-      [==>..................]  recovery = 12.6% (37043392/292945152) finish=127.5min speed=33440K/sec
-
-unused devices: <none>
diff --git a/test/recovery.output b/test/recovery.output
deleted file mode 100644
index 44c3596..0000000
--- a/test/recovery.output
+++ /dev/null
@@ -1 +0,0 @@
-WARNING: Missing 1 of 6 devices. Recovering: progress=12.6% finish=127.5min speed=33440K/sec | 'raid level'=5 'size in blocks'=1464725760 'num devices'=6 'num failed devices'=0;;1;0;6 'num spare devices'=0;;;0;6 'recovery progress'=12.6%;0;;0;100 'device 0: sdc1'=0 'device 1: sdd1'=1 'device 2: sde1'=2 'device 3: sdf1'=3 'device 4: sdg1'=4 'device 6: sdh1'=6
diff --git a/test/run.sh b/test/run.sh
index c5302b8..e7f6699 100755
--- a/test/run.sh
+++ b/test/run.sh
@@ -1,9 +1,10 @@
 dir="`dirname $0`"
 
 ERROR=false
-for f in `ls $dir/*.input`; do
+IFS=$'\n'
+for f in `find $dir -type d -name '*.input'`; do
 	RESULT="`$dir/../check_mdstat.sh --raid md0 --input $f`"
-	EXPECTED="`cat $dir/$(basename $f .input).output`"
+	EXPECTED="$(<$dir/$(basename $f .input).output)"
 	if [ "$RESULT" != "$EXPECTED" ]; then
 		echo "Error for test $(basename $f .input):"
 		echo "	Expected"