------------------------------------------------------------------------------- Hints and Tips for general shell script programing Shell Tutorial Video for begineeres... http://ontwik.com/linux/shell-programming-for-beginners/ ------------------------------------------------------------------------------- Where is this script located WARNING: this will fail if the user is playing with $0 For example using a symbolic or hard link with a unexpected name. =======8<-------- #!/bin/sh # simplist -- a little too simple PROGNAME=`basename $0` base name of program =======8<-------- =======8<-------- #!/bin/sh # little more complex... PROGNAME=`type $0 | awk '{print $3}'` # search for executable on path PROGNAME=`basename $PROGNAME` # base name of program =======8<-------- =======8<-------- #!/bin/sh # Advanced... # Script name, in what directory, and in what directory is user # running the script from. # # Discover where the shell script resides PROGNAME=`type $0 | awk '{print $3}'` # search for executable on path PROGDIR=`dirname $PROGNAME` # extract directory of program PROGNAME=`basename $PROGNAME` # base name of program # Fully qualify directory path (remove relative components and symlinks) ORIGDIR=`pwd -P` # original directory PROGDIR=`cd $PROGDIR && pwd -P || echo $PROGDIR` # program directory # NOTE: Bash's "pwd" builtin does not resolve symbolic links, as it returns # the 'logical' working directory, also stored as the environment variable # $PWD. But using "pwd -P" works whether it is a shell builtin, or a actual # binary command (which ignores the argument). =======8<-------- Results... $ORIGDIR -- where the users was when called $PROGDIR -- script directory location (and now current directory) $PROGNAME -- This scripts executable name without directory component Note: short cuts for bash PROGNAME=`type $0 | awk '{print $3}'` PROGDIR="${PROGNAME%/*}" # get directory component (remove short match) PROGNAME="${PROGNAME##*/}" # get basename component (remove long match) ------------------------------------------------------------------------------- Shell Script Option Handling # Old, simple, and typical Usage Handling.. # PROGNAME=`basename $0` # Or a full script locator (below) Usage() { echo >&2 "$PROGNAME:" "$@" echo >&2 "Usage: $PROGNAME [options] [file...]" exit 10 } # ---------------------- # My Shell Usage and Option handing #!/bin/sh # # script [options] args... # # The 'Usage()' and 'Help()' functions uses $PROGDIR to find this actual # script, then reads and output these comments (short usage or full set of # comments) as the documentation for this script. That is the script # commands and usage documention are in the same file, making it self # documenting, via options. # ### # # programmers only docs, which are not output by Usage() # # Discover where the shell script resides PROGNAME=`type $0 | awk '{print $3}'` # search for executable on path PROGDIR=`dirname "$PROGNAME"` # extract directory of program PROGNAME=`basename "$PROGNAME"` # base name of program # Fully qualify directory path (remove relative components and symlinks) ORIGDIR=`pwd -P` # original directory PROGDIR=`cd "$PROGDIR" && pwd -P || echo "$PROGDIR"` # program directory cd "$ORIGDIR" Usage() { # Report error and Synopsis line only echo >&2 "$PROGNAME:" "$@" sed >&2 -n '1,2d; /^###/q; /^#/!q; /^#$/q; s/^# */Usage: /p;' \ "$PROGDIR/$PROGNAME" exit 10; } Help() { # Output Full header comments as documentation echo >&2 "$PROGNAME:" "$@" sed >&2 -n '1d; /^###/q; /^#/!q; s/^#//; s/^ //; p' \ "$PROGDIR/$PROGNAME" exit 10; } Error() { # Just output an error condition and exit (no usage) echo >&2 "$PROGNAME:" "$@" exit 2 } # minimal option handling while [ $# -gt 0 ]; do case "$1" in # Standard help option. -\?|-help|--help|--doc*) Help ;; --) shift; break ;; # forced end of user options -*) Usage "Unknown option \"$1\"" ;; *) break ;; # unforced end of user options esac shift # next option done # --------------------------- # # Option handling (full examples of option types) # while [ $# -gt 0 ]; do case "$1" in # Standard help option. -\?|-help|--help|--doc*) Help ;; # Simple flag option -d) DEBUG=true ;; # Word flag option -debug) DEBUG=true ;; # Simple option and argument EG: -n name -n) shift; name="$1" ;; # Option and argument joined EG: -Jname -J) name=expr + "$1" : '-.\(.*\)'` || Usage "Option \"$1\" missing required value" ;; # Joined OR unjoined Argument EG: -Nname or -N name -N*) Name=`expr + "$1" : '-.\(..*\)'` || { shift; Name="$1"; } ;; # As last, but if -b0 is an posibility then you need to use this instead. # Otherwise it will think the value is unjoined when it isn't Arrrgghhhh... -b*) bits=`expr + "$1" : '-.\(..*\)'` [ "$bits" ] || { shift; bits="$1"; } ;; # Number with type check EG: -{size} -[0-9]*) size=`expr + "$1" : '-\([0-9]*\)$'` || Usage [ "$size" -eq 0 ] && Usage Width=$size; Height=$size ;; # Numbers with type checks EG: -{width}x{height} -[0-9]*x[0-9]*) w=`expr + "$1" : '-\([0-9]*\)x'` || Usage "Bad Geometry" h=`expr + "$1" : '-[0-9]*x\([0-9]*\)$'` || Usage "Bad Geometry" [ "$width" -eq 0 -o "$height" -eq 0 ] && Usage "Zero Geometry" geometry="${w}x${h}" ;; # rectangular crop EG: WxH+X+Y [0-9]*x[0-9]*+[0-9]*+[0-9]*) CROP="$1" ;; # Generalised Argument Save EG: -G value => $opt_G -*) var=`expr + "$1" : '-\(.\).*'` case "$var" in [a-zA-Z]) arg=`expr + "$1" : '-.\(..*\)'` || { shift; arg="$1"; } eval "opt_$var"="\"\$arg\"" ;; *) Usage "Bad Non-Letter option \"$1\"" ;; esac ;; -) break ;; # STDIN, end of user options --) shift; break ;; # forced end of user options -*) Usage "Unknown option \"$1\"" ;; *) break ;; # unforced end of user options esac shift # next option done # Handle normal arguments now... FILE="${1:-/dev/stdin}" [ ! -f $FILE ] && Error "Input \"$FILE\": does not exists" [ ! -r $FILE ] && Error "Input \"$FILE\": is not readable" # or [ $# -lt 2 ] && Usage "Too Few Arguments" [ $# -gt 2 ] && Usage "Too Many Arguments" ------------------------------------------------------------------------------- Dependency Program Testing DEPENDENCIES="sed awk grep egrep tr bc magick" # Check Dependencies a script requires is available for i in $DEPENDENCIES; do type $i >/dev/null 2>&1 || Usage "Required program dependency \"$i\" missing" done ------------------------------------------------------------------------------- Variable testing under shell (rules of thumb) Rules of thumb... * Always quote all variables being tested * Only use the boolean type test (below) when you have complete control of all posible settings. * Prepend X (or other alphanumberic) when comparing unknown strings. * Don't use ! if you can avoid it (too many changes between shells) Boolean Variable tests (variable contents must be fully controlled)... true="true" false="" [ "$var" ] && echo var is true [ -z "$var" ] && echo var is false Test of variables containing ANY string (see PROBLEM CASES below) option=-xyzzy [ "X$option" != X ] && echo option is defined [ "X$a" = "X$b" ] && echo "$a equals $b" IS a variable defined... if [ ${#var[@]} -eq 0 ]; then echo "Variable/Array is undefined" elif [ "X$var" = "X" ]; then echo "Variable is a NULL string" else echo "Variable has non-null value of \"$var\"" fi # Note: There seems to be no way to tell if a special element # of an array is defined, without requesting a 'key' list. # Remember bash arrays are numbered (not hashed), # but are also sparse arrays (see below). PROBLEM CASES... These cases cause the "[...]" tests to fail badly! [ $var ] but var="a = b" [ "$var" ] but var=-t (actually any option starting with '-') [ "$a" = "$b" ] but a='(' and b=')' This is why you must use the string test above (with 'X' prefixes). NOTE test ! ... differs from UNIX to UNIX Under BASH, it is the NOT of the next 3 arguments that follow (non-sensical). Under Solaris SH, it seems to handle precedence correctly. EG: test "" -a ! "" is false as you would expect under BASH BUT test ! "" -a "" is true for BASH and false for Solaris SH ------------------------------------------------------------------------------- Shell Mathematics EXPR.... This is the traditional way (integer) all elements must be separate arguments, and special charcaters like '*' (for multiply) must be escaped. expr + 5 / 2 2 BASH... In BASH you can use some special constructs (no $ prefix needed) Specifically ((...)) for conditionals (eg: exit code testing) and $((...)) for variable substitution. $(...) for command sunstitution. Note $[...] is an older construct and is depreciated. a=1 if (( a == 0 )); then echo true; else echo false; fi a=5 b=2 echo $(( a/b )) 2 echo $( expr + 5 / 2 ) # using bash with expr 2 KSH... The [[...]] was introduced by ksh88 as a [...] replacement and should be used for string comparisions, while ((...)) are for numerical comparisions. You can still use [...] for the older traditional test command in ksh88 BC... echo 5/2 | bc 2 BC With scaling echo 'scale=5; 5/2' | bc 2.50000 BC with a -l flag (floating point and math functions) This also enables scaling precision (scale=20) echo 5/2 | bc -l 2.50000000000000000000 echo 'a(1)*4' | bc -l 3.14159265358979323844 ImageMagick... It is a rather long winded, but if you are doing imaging! convert null: -precision 10 -format '%[fx:atan(1,1)*4]' info: 3.141592654 Built-ins are faster, though not as portiable time for I in $(seq 1 1000); do A=$((1+1)); done real 0m0.009s time for I in $(seq 1 1000); do expr 1 + 1 > /dev/null; done real 0m1.330s ------------------------------------------------------------------------------- Expr "match" failure This works reasonably well - most of the time word=abcde expr "$word" : ".\(.*\)" bcde But WILL fail for the word "match" word=match expr "$word" : ".\(.*\)" Also using the "match" form will fail word=match expr match "$word" ".\(.*\)" expr: syntax error The problem is "match" is a keyword, which will confuse "expr" argument handling. The solution is to always start "expr" with '+' word=match expr + "$word" : ".\(.*\)" atch This Cavat can be very important for ALL expressions, NOT just a string test. GNU "expr" has a special option to prevent this problem, but that is not portible. MacOsX "expr" using "match" however will fail (it is not understood! expr match "this_is_a test" 'this' expr: syntax error You must use the ':' sub-string test instead expr "this_is_a test" : 'this' 4 Unfortunatally adding the '+' to the start also fails Arrggghhhh.... expr + "this_is_a test" : 'this' expr: syntax error I have not found a solution that works on BOTH linux and MacOSX for all input! ------------------------------------------------------------------------------- Argument Sorting By Shells All Shells normally sort the output of any commands with meta characters. Also "ls" will sort its arguments again internally, while "echo" will not. EG compare the output of these commands ls -d b* a* echo b* a* echo [ba]* The "ls" sorts its output so a's are before b's, the second does not. However the shell sorts the arguments of the third so a's are again first. However the "{}" meta characters do NOT sort the results... echo {b,a}* will output b's first then a's. If 'ls' is used instead of 'echo', 'ls' will sort results. This can be useful when the order of the arguments is important. This has been tested and works as described in csh, tcsh, bash, and zsh NOTE: 'dash' does not have this functionality. ------------------------------------------------------------------------------- Argument handling... If you care for the possibility that there aren't any arguments. "$@" -> "" (one empty argument) ${1+"$@"} -> nothing at all This is not a problem with newer modern shells, only very old Bourne shells. WARNING: Command line shell argument start at 0! sh -c 'printf " <%s>" "$@"' 1 2 3 4 5 or sh -c 'printf " <%s>" "$@"' `seq 5` Will print <2> <3> <4> <5> Note that the first arguement is missing. That is because it was assigned to argument $0 and not $1 or the $@ and $* variables! To use all the arguments on the shell command line, use either sh -c 'printf " <%s>" "$@"' -- `seq 5` or sh -c 'printf " <%s>" $0 "$@"' `seq 5` the later is not recomended as if no arguments are provided $0 defaults to "sh" instead of the empty string! This is a postix compliant feature, $0 being the program name when a shell script is called. The "-c" command line script just layers on top of this. this is the case with solaris bourne sh, bash, ksh, and zsh. Csh and tcsh shell does not have an equivelence. See "find" and its "-exec" option, (see below), for a useful example of this. I also use it in csh aliases to provide a default argument. alias xvd 'sh -c '\''cd ${1:-.}; xv -vsmap &'\'' -- !:* &' ------------------------------------------------------------------------------- Checking for a numerical value Built-ins only... case "$var" in '' | *[!0-9]*) echo "non-numeric" ;; *) echo "numeric" ;; esac # --- Nick Holloway (alfie@dcs.warwick.ac.uk) Using egrep... echo "$arg" | egrep '^[0-9]+$' >/dev/null || echo "not-numeric" Using expr. (fails for 'zero' input value). expr match "$var" '\([0-9]*\)$' || echo "not a non-zero-numeric value" ------------------------------------------------------------------------------- Auto change shells on brain dead systems (Ultrix) #!/bin/sh - # # Check the type of shell that is running (For Ultrix) [ "X$1" != 'X-sh5' -a -f /bin/sh5 ] && exec /bin/sh5 -$- "$0" -sh5 "$@" [ "X$1" = 'X-sh5' ] && shift # ------------------------------------------------------------------------------- Command search (bash) 1/ REDIRECTION is done (before anything else!) 2/ simple alias expandsion 3/ Parameter expansion, command substitution, arithmetic expansion, and quote removal, variable assignment performed if '=' found. 4/ functions exectuted 5/ shell built-in commands 6/ lookup command in hash table (error if it is in hash but does not exists) 7/ search the command PATH variable 8/ else error "Command not found" ------------------------------------------------------------------------------- One line if-then-else shell command cmd1 && cmd2 || cmd3 This however will execute cmd3 if EITHER cmd 1 or cmd2 fails! If cmd2 never fails - no problems This means a status check can be written as command && echo TRUE || echo FALSE As the first echo is always true, so the last echo will only work if the first command was false. ------------------------------------------------------------------------------- Results from commands a=`command` a=$(command) Or directly split into words... read -r a b c < <( command ) Equivelent to the use of a pipline using a named pipe mknod -p cmd_pipe echo one two three > cmd_pipe & read -r a b c < cmd_pipe rm -f cmd_pipe echo "a=\"$a\" b=\"$b\" c=\"$c\"" You can also directly read into a bash array (sequential)... array=( $( command ) ) Split by column (read comma separated CSV file) OIFS=$IFS IFS=',' while read a b c do echo "LOGIN:$a LASTNAME:$b FIRSTNAME:$c" done < fileA.csv Read results line by line { read -r line1 read -r line2 read -r line3 } < <( command ) For array of lines... unset arr i while read -r; do arr[i++]=$REPLY done < <(your command) # Now get any final line that has no final newline (may be empty) [[ $REPLY ]] && arr[i++]=$REPLY ------------------------------------------------------------------------------- Break string into words... Bourne shell "set" method with IFS save. (Only shell built-ins!) WARNING: this replaces script/function args. (Dangerious) (NOTE: no quote in the 'set' - making it very dangerious security-wise) line="one two three" set -- $line a="$1" b="$2" c="$3" echo "# a=\"$a\" b=\"$b\" c=\"$c\"" # a="one" b="two" c="three" Using IFS to change separator character line="one:two:three" OIFS=$IFS; IFS=":"; set -- $line; IFS="$OIFS" a="$1" b="$2" c="$3" echo "# a=\"$a\" b=\"$b\" c=\"$c\"" # a="one" b="two" c="three" Using separate variable assignment and sed/awk/greps (very slow - but can extract values from very complex lines) line="one:two:three" a=`echo "$line" | awk -F: '{print $1}'` b=`echo "$line" | awk -F: '{print $2}'` c=`echo "$line" | awk -F: '{print $3}'` echo "# a=\"$a\" b=\"$b\" c=\"$c\"" # a="one" b="two" c="three" OR line="one:two:three" a=`echo "$line" | sed 's/:.*//'` b=`echo "$line" | sed 's/.*:\(.*\):.*/\1/'` c=`echo "$line" | sed 's/.*://'` echo "# a=\"$a\" b=\"$b\" c=\"$c\"" # a="one" b="two" c="three" This technique allows you to read ALL the output of the command into a variable and extract different parts and values as needed from the multi-line results. Bash 'here string' reading (Safer - Bash built-ins only) var="one two three" read -r a b c <<< "$var" echo "# a=\"$a\" b=\"$b\" c=\"$c\"" # a="one" b="two" c="three" With field separators, including blank fields... IFS=":" read -r a b c <<< "hello::world" echo "# a=\"$a\" b=\"$b\" c=\"$c\"" # a="hello" b="" c="world" or you can pre-process the line so you can skip fields at start middle or end read -r a b c <<< $(echo "::hello::world::" | tr ':' ' ') echo "a=\"$a\" b=\"$b\" c=\"$c\"" # a="hello" b="world" c="" WARNING: IFS normally allows multiple spaces between fields (whitespace). But if it is not the whitespace default setting, then the fields are assumed to be separated by exactly one character. As such in one example $b is a empty string! (Awk is similar) BUT in if mapped to white space it $b is not empty and $c is empty It is also equivelent to direct reading from a command using... read -r a b c < <( echo one two three ) echo "a=\"$a\" b=\"$b\" c=\"$c\"" (see next section) Bash Alturnative, using regular expressions (I can't get this to work!) foo="abc-start-ijk-end-end" [[ "$foo" =~ "start-(.*)-end" ]] && middle="${BASH_REMATCH[1]}" echo "BASH_REMATCH=\"$BASH_REMATCH\"" echo "middle=\"$middle\"" BASH_REMATCH is an special built-in array 0 = whole matched expression n = positional sub-expression ------------------------------------------------------------------------------- LIST functions (plain Bourne Shell) (Also usable for PATH-like environment variables) NOTE the use of sub-shells (...) to scope the IFS setting list="a:b:c:d" # list variable element="e" # element of the list sep=":" # list_seperator (not used, and not white space) Append to list (even if empty!) list="${list:+$list$sep}$element" Split up list function split_list () { ( IFS="$sep" set -- $list for f in "$@"; do echo -n "${f:-.} " done; echo ) } Loop over list for item in `( IFS="$sep"; echo $list; )`; do echo item=$item done Count of elements count=`IFS="$sep"; set - $list; echo $#` get I'th element (shell must understand shift argument) element=`IFS="$sep"; set - $list; shift $I; echo $1` get first element element=`IFS="$sep"; set - $list; echo $1` delete first element #list=`echo "$list" | sed "/$sep/"\!"d; /:/s/^[^$sep]*$sep//;"` list=`echo "$list" | cut -d"$sep" -f2-` delete specific element over the whole list -- not right??? list=`echo "$list" | sed "s|${element}${sep}||g; s|${sep}${element}\$||;"` Of course BASH arrays superseeds all this. ------------------------------------------------------------------------------- Bash Array Example Also see http://mywiki.wooledge.org/BashFAQ/005 NOTES: * Indexes start at zero * The zeroth element is also the value of the variable of the same name. * Elements do not need to exist (sparse array) but are "" if referenced, * Referencing an undefined element, does NOT define the element. Pre-delcaring an array (not actually needed) declare -a xyz[50] Initialize letters=( a b c d e f ) # just define in sequence elements=( [0]=x [1]=y [3]=z ) # element 2 is not defined letters[6]=g # add/change a element echo "${letters[4]}" # referencing an element echo "${letters[@]}" # list all elements Read file into an array words=( $(cat file) ) # you do not need to worry about quotes # or [#] in the file. Due to quote order echo ${#words[@]} # Number of words (not sparse) Append onto array list=( "${list[@]}" "new_element" "another_element" ) Copy Array new_array=( "${old_array[@]}" ) new_array="${old_array[@]}" Loop List array directly for i in "${elements[@]}"; do echo "'$i'" done Loop List array by index for i in "${!elements[@]}"; do echo "$i : '${elements[i]}'" done The zeroth element is the same as the variable name without the index. variable[0]=zero echo $variable zero variable=different echo ${variable[0]} different echo ${#variable} # length of zeroth element (0=unset or null) 8 Sparse Array Elements sparse[4]="y" # create sparse array of values sparse[7]="n" sparse=( [4]="y" [7]="n" ) # alternative method (equivelent) echo ${sparse[4]} # get a spefic value y k=7 echo ${sparse[k]} # offset is an arithmetic expansion n sparse[k]=q # subsitute value echo ${sparse[k]} q sparse[22]="nan" # add another parameter echo ${sparse[*]} # list all parameters y q nan sparse[13]='' # add null value in sparse array for i in "${sparse[@]}"; do # quoted expansion echo "\"$i\"" done "y" "q" "" "nan" compress=( "${sparse[@]}" ) # compress arry removing missing elements echo ${compress[3]} nan echo ${#sparse[@]} # Number of defined elements 4 echo ${!sparse[*]} # what indices are in use 4 7 13 22 echo ${!compress[*]} 0 1 2 3 echo "${sparse[@]:5:3}" # list next 3 items after index 5 q nan # note the extra space in output for NULL echo ${#sparse[22]} # Length of a specific element 3 unset sparse[13] # undefine a element echo ${#sparse[@]} 3 unset sparse # undefine whole array Array Stack... (using the first elemant as a stack count) unset stack # initialise or reset (no number, until used) stack=0 # initialize with a actual number of elements stack[++stack]="'$stack'" # PUSH some value (repeat this a few times) unset stack[stack--] # POP - you can over pop without problems! # all that does is replace 0 with empty string echo $stack # number of elements echo ${stack[stack]} # get top element from stack echo ${stack[stack--]} # get and pop top elemement # WARNING: element is not actually removed! echo ${stack[stack]} # Get top element unset ${stack[stack--]} # actually remove the top element from stack! echo "${stack[@]}" # get stack size and all the elements on stack # list includes 'pop'ed elements, now forgotten Alternative see... See http://www.tldp.org/LDP/abs/html/arrays.html#STACKEX Sort an array (shell built-ins only) =======8<-------- #!/bin/bash i=0 for n in "$@"; do a[i++]="$n" done # Bubble Sort Array for (( i=0; i<(${#a[@]}-1); i++ )); do for (( j=i+1; j<(${#a[@]}); j++ )); do if [[ ${a[j]} < ${a[i]} ]]; then t=${a[j]}; a[j]=${a[i]}; a[i]=$t; fi done done # Output the array echo "${a[@]}" =======8<-------- QUESTIONS.... 1/ How can you find out if a specific index is actually defined? That is it isn't just an empty string! The only way seems to be search via elements or indexes! or use multiple ${ :- } type tests (See "Variable Testing" above for simple (non-array) variables) 2/ How can you get just the top most element in a sparse array? That is without listing all the indices in array. ------------------------------------------------------------------------------- builtin cat command This cat only uses the shell builtins! As such can be used on a machine which has no access to shared libraries and nothing but the statically linked bourne sh can run. shcat() { while test $# -ge 1; do while read i; do echo "$i" done < $1 shift done } Of course the real cat command is only as big as it is, to protect the user from himself and to provide a huge number of options. PS: If the ls command is also not available then you can use echo * to do a directory listing using only builtins. Though this will not tell you what files are executables or sub-directories, or handle problems like spaces in filenames, or unprintable characters. ------------------------------------------------------------------------------- The useless use of 'cat'! You often see in shell scripts... cat input_file | some_command The cat is useless as it is exactly the same as some_command < input_file without needing to fork the extra "cat" process or creating a pipeline. However it is sometimes useful to do anyway to make the code more readable. Particularly in long pipelines, or where "some_command" is a large while loop. WARNING: "some_command" will be exectuted as a sub-shell in the first instance, but it is the main-shell in the second instance. ------------------------------------------------------------------------------- Using HERE files in a pipeline of commands... command < mypipe & while read line; do foo=something done < mypipe echo "foo = $foo" Bash can also use builtin named pipes. foo=nothing while read line; do foo=something done <( echo "testing 1 2 3" ) echo "foo = $foo" See http://mywiki.wooledge.org/BashFAQ/024 See also co-processes.hints http://www.imagemagick.org/Usage/info/shell/co-processes.hints on bi-directional I/O to a background process, daemon, or internet service. ------------------------------------------------------------------------------- Newlines in Command Substitutions... Note that var=`cmd` does save the newline characters. But to use then you must quote the variable! :::> m=`mount | awk '{print $1}'`; :::> echo $m / /usr /home :::> echo "$m" / /usr /home In other words outside quotes newlines in the input are treated purely as white space between arguments and thus ignored. Inside quotes newlines are retained as newlines. ------------------------------------------------------------------------------- Inserting a command line inside a shell script... For example embedding, sed, awk or perl scripts in a shell script. The normal way is to use Single Quotes around the command line script. For Example... awk '# add prefix to lines { print "prefix> " $0 } ' list.txt Note the whole script is inside single quotes on the nawk command line! ASIDE: old versions of awk must have something on the first line thus the addition of the # comment to keep it happy! Perl needs no such comment but does require a -e option to execute a command line argument. Environment Variables Both "awk" and "perl" can read environment variables for data passing. export column=2 awk 'BEGIN { column = ENVIRON["column"] } /^[^#]/ { print $column } ' file Insert a Shell variable in quoted code... To include an external shell variable, you need to switch quote modes just for that variable... file="list.txt" awk '# add prefix to lines { print "'"$file"'> " $0 } ' "$file" For awk you can declare the variable on the command line, file="list.txt" awk -v file="list.txt" '# awk code { print file "> " $0 } ' "$file" OR file="list.txt" awk '# awk code { print file "> " $0 } ' file="$file" "$file" Note the latter may or may not have the 'file' variable available to the BEGIN block. Single Quote Insertions... There are a number of ways... awk '# print single quotes BEGIN { print "'\''" print "\047" q="\047"; print "Can or can" q "t?" exit }' The first works for any command line script. The rest are "awk" (or whatever) dependant. You can also use the special $'...' form of single quote delimited strings which is specific BASH and KSH, (ansi-c style quoting). That is force all backslashed escapes to be performed. EG: All backslashes are handled as be the ANSI-C compilion of strings. EG: \n \t \' \" all handled. echo $' single_quote=\' double_quote=\" ' CAUTION: Watch for single quotes inside any COMMENTS in the script code block! Comments are within the single quotes so are scanned for those quotes. Alternative Method for code blocks in shell scripts. Perl has the added advantage that you can pipe the command to execute into it. For example using a 'herefile' perl <<'EOF' print "Hello, world from perl in a here document.\n"; EOF Of course you have less control over the escaping and addition of external variables and values into the script. Also see HERE files in "file.hints". http://www.imagemagick.org/Usage/info/shell/file.hints. CSH SCRIPTS: If you must do this in a csh script (don't do it!), you will need to escape (backslash) the new line at the end of every line, even though single quotes are being used. Also watch out for history escapes '!' which in csh work inside single quotes! ----------- Warning: Some precautions are necessary with the contents of the variable, as the inserted value will be parsed by the command (awk/sed/perl/etc..). In perl characters than may need special handling include $ @ " ' \ and so on. A better method may be to 'pipe' the variable into the command For example this works... encode() { # WWW url encoding (typically needed for passwords) echo -n "$1" | perl -e '$_ = <>; s/[^\w.]/sprintf "%%%02x",ord $&/eg; print;' } This 'standard way' will fail on characters such as '@' or '$' or '"' encode() { # WWW url encoding (typically needed for passwords) perl -e '$_ = "'"$1"'"; s/[^\w.]/sprintf "%%%02x",ord $&/eg; print;' } This is especially important for 'tainted' input data. ------------------------------------------------------------------------------- Is COMMAND available There is two techniques available to test if a command is available. The first is to use the `status' return of the "type" or "which" command of the shell you are using. The Second is to examine the output of that command itself. Using status return. This is a simple method of testing the existance of a command but DOES NOT WORK ON ALL SYSTEMS! The problem is that old shells (For Example: SunOS bourne-sh) always returns a true status weather the command is present or not! Bourne Shell if type COMMAND >/dev/null 2>&1; then # COMMAND is available fi C-Shell Warning: The "which" command in C shell is not a builtin but a external script which sources your .cshrc (YUCK). As such the bourne shell alias is prefered which will only search your current command PATH. if ( ! $?tcsh ) then alias which 'sh -c "type \!:1 2>&1"' endif if ( -x "`which COMMAND >/dev/null`" ) then # COMMAND Available endif TC-Shell Tcsh 6.06 also does not return the correct status in its which command. Use it like the csh which above. WARNING: this method will also test positive for :- subroutines, bash aliases, and probably other non-command definitions. Examine output The other am more reliable method is to examine the output of the "type" or "which" command to looks for the string "not found" (See below). This is more complex than the above status check but should work on ALL unix machines regardless of age. Bourne Shell cmd_found() { case "`type $1 2>&1`" in *'not found'*) return 1 ;; esac; return 0 } # ... if cmd_found COMMAND; then # COMMAND is available fi # WARNING: Do not use... cmd_found COMMAND && COMMAND args & # As it leaves behind a waiting sub-shell C-Shell & Tcsh (See notes below) if ( ! $?tcsh ) then alias which 'sh -c "type \!:1 2>&1"' endif ... if ( "`which less`" !~ *'not found'* ) then # COMMAND Available endif NOTES for "which/type" commands :- The above methods look for the specific string "not found" This is important as the sh type command and tcsh which command produce different output, and this may also vary from bourne shell to bourne shell or other shell types. Csh -- "which" is an unreliable shell script! fudge it into a shell script "type" command. See the "Which Problem" below. Tcsh > which less /opt/bin/less > which junk junk: Command not found. > which which which: shell built-in command. > alias a alias > which a a: aliased to alias Solaris Bourne shell $ type less less is /opt/bin/less $ type junk junk not found $ type type type is a shell builtin $ func(){ echo ha; } $ type func func is a function func(){ echo ha } Solaris Ksh As per Sh, but the actual function definition is NOT listed Bash $ type less less is /opt/bin/less $ type junk bash: type: junk: not found $ type type type is a shell builtin $ func(){ echo ha; } $ type func func is a function func () { echo ha } NOTE: bash also has a type -t which responds with a single word "file", "alias", "function", "builtin", "keyword", or nothing if command does not exist. A -p will print the disk file name, or nothing. A -a prints all the places that have that name. From the results above, only the appearence of "not found" in the false statement is consistant. But only when the result is not a bourne shell function, which presumably replaces the real-command of that name. The Expanded Bourne shell form, without using the "cmd_found" function is as follows, But is is a low simpler and easier to read if you use the funtion. If command present if expr + "`type COMMAND 2>&1`" : '.*not found' == 0 >/dev/null; then # COMMAND is available fi and its inverse (not present) if expr + "`type COMMAND 2>&1`" : '.*not found' >/dev/null; then # Comand is NOT present fi finally only using built-in commands... case "`type COMMAND`" in *'not found'*) # Command not found ;; *) # Command found ;; esac Functional forms cmd_found() { expr + "`type $1 2>&1`" : '.*not found' == 0 >/dev/null } OR cmd_found() { case "`type $1 2>&1`" in *'not found'*) return 1 ;; esac; return 0 } ------------------------------------------------------------------------------- CSH Which problem! On SunOS: The which command is a csh script that specifically reads the .cshrc file to find out about aliases. To avoid having .cshrc do lots of odd things to your script, use the following form. set program = `/bin/env HOME= /usr/ucb/which $program` This is NOT a problem in Tcsh, where "which" is a built-in. The best solution it to replace "which" with the bourne shell "type" command in C shells but leave it alone for TC shells. if ( ! $?tcsh ) then alias which 'sh -c "type \!:1 2>&1" | sed "s/.* is //"' endif set program = `which $program` ------------------------------------------------------------------------------- echo without a return (bsd & sysV) For Bourne Shell # echo without a return (bsd & sysV) if [ "X`echo -n`" = "X-n" ]; then echo_n() { echo ${1+"$@"}"\c"; } else echo_n() { echo -n ${1+"$@"}; } fi For Csh or Tcsh (which you shouldn't use for scripts anyway) # echo without a return (bsd & sysV) if ( "X`echo -n`" == 'X-n' ) then alias echo_n 'echo \!:* "\c"' else alias echo_n 'echo -n \!:* ' endif For all systems, but uses non-builtin shell commands echo '----- Press return to continue. ' | tr -d '\012' ------------------------------------------------------------------------------- Print a line of N repeated characters This is weird but it works... awk 'BEGIN{$79=OFS="*";print}' Function method - no newline... function repeat(){ for x in $(seq $2); do echo -n "$1" done } repeat '*' 6 And this can output a multi-character repeat echo "<-`repeat '-O-' 6`->" ------------------------------------------------------------------------------- Looping and Generation of a list of numbers Bourne Shell while loop limit=10; n=1; while [ $n -le 10 ]; do echo $n; n=`expr $n + 1`; done with number formatting (printf) limit=10; n=1; while [ $n -le $limit ]; do printf %03d $n; n=`expr $n + 1`; done with number formating (sed truncation) NOTE: number of . in final parenthesis is the final size But this will fail if number is larger limit=10; n=1; while [ $n -le $limit ]; do echo "000000$i" i=`expr $i + 1` done | sed 's/.*\(...\)$/\1/g' Bash allows a C-like for loop... for (( i=1; i <= 10; i++ )); do echo $i; done Note: That both the shell "while" and the Bash "for (( ; ; ))" method will do a final incrment of the variable, before exiting. That means after the loop you can get some idea if the loop completed, or a "break" was used. Only on completion will the number be larger than the range {end}. All bets are off however if the index number is modified inside the loop to cause the break. OR a "for .. in" loop is used instead (see below) Bash (versions > 3.2) - not the case for MacOSX echo {5..15} # forward can be negative numebrs 5 6 7 8 9 10 11 12 13 14 15 echo {15..5} # reversible 15 14 13 12 11 10 9 8 7 6 5 echo {5..15..2} # increment supported in BASH v4.0+ 5 7 9 11 13 15 echo {0001..5} # zero padding 0001 0002 0003 0004 0005 echo {a..d} # single letters (only singles) a b c d echo {c..a}{A..C} # or letter sequence (multiple expand) cA cB cC bA bB bC aA aB aC echo {{1..5},{23..27}} # two ranges of numbers 1 2 3 4 5 23 24 25 26 27 WARNING: you can not use variables in this syntax! This for example it will NOT work ... {1..$limit} limit=5 echo {1..$limit} But this will work (painful though it is)... echo $(eval echo {1..${MAX_SEGMENTS}}) You better of using a "for( ; ; )" loop! Awk awk "BEGIN {for (i=1;i<$COUNT;i++) {printf(\"%03d\n\",i)} }" Perl perl -le 'for(1..10){print}' Dev Zero -into-> newlines -into-> cat line counting Specifically dd | tr | cat This reads 10 (or however many) 'nulls' from /dev/zero, then converts that into 10 return characters, which cat numbers, finally any non-return white space is removed. dd 2>/dev/null if=/dev/zero bs=10 count=1 | tr \\0 \\012 |\ cat -n | tr -d '\40\11' seq The Gnu "seq" command is the simplest and most wide spread... Args: {start} [[{incr}] {end}] Note that {incr} is in middle which is logical but can be awkward. seq 10 seq 5 10 seq 10 -1 1 seq 1 .1 2 seq -f %03g 5 5 100 As you can see it can go forwards, from any range, with any increment, even backwards, or using factions (floats), and with format control. WARNING: "seq" is not available under MacOSX :-( See 'seq' replacement function below (and in my scripts area) jot It seem "jot" can also be used, but it is not a standard install in fedora. The first number by default is the total number of numbers wanted spread out equally accross the range given (even floating point) jot 3 # 1 2 3 jot 2 12 # 12 13 14 jot 3 12 16 # 12 14 16 jot 4 12 17.0 # 12.0 13.7 15.3 17.0 jot -p4 4 12 17.0 # 12.0000 13.6667 15.3333 17.0000 Using '-' will let it work more like "seq" using args: {start} {end} {incr} jot - 2 10 2 # 2 4 6 8 10 It can create random numbers jot -r 5 1000 9999 # 3435 1811 2641 7472 2528 or repeat a specific word or string jot -b word 3 # word word word DIY seq() replacements # seq_count {end} seq_count() { i=1; while [ $i -le $1 ]; do echo $i; i=`expr $i + 1`; done } # simple_seq {start} {incr} {end} simple_seq() { i=$1; while [ $i -le $3 ]; do echo $i; i=`expr $i + $2`; done } # simple_integer [-f format] [{start} [{incr}]] {end} # almost perfect but only with increasing intergers seq_integer() { if [ "X$1" = "X-f" ] then format="$2"; shift; shift else format="%d" fi case $# in 1) i=1 inc=1 end=$1 ;; 2) i=$1 inc=1 end=$2 ;; *) i=$1 inc=$2 end=$3 ;; esac while [ $i -le $end ]; do printf "$format\n" $i; i=`expr $i + $inc`; done } # float_seq --- FUTURE (when needed) See also "Columns of Data" (below) for more examples... ------------------------------------------------------------------------------ Arrays of Numbers You can make an array say A-1 to C-3 easilly in bash using echo {A..C}-{1..3} A-1 A-2 A-3 B-1 B-2 B-3 C-1 C-2 C-3 You can even add prefix or postfix strings echo s {A..C}-{1..3} e s A-1 A-2 A-3 B-1 B-2 B-3 C-1 C-2 C-3 e But what if you want a prefix/postfix string around each row that gets a lot more complicated... Lets make this a bit clearer, which make working it out easier echo Array \ ' s' A-{1-3} 'E ' \ ' s' B-{1-3} 'E ' \ ' s' C-{1-3} 'E ' \ End Array s A-{1-3} e s B-{1-3} e s C-{1-3} e End So we need to reproduce that by merging the lines echo Array \ '" s" '{A..C}'_{1..3} "e "' \ End Array " s" A_{1..3} "e " " s" B_{1..3} "e " " s" C_{1..3} "e " End and evaluate it eval echo Array \ '" s" '{A..C}'_{1..3} "e "' \ End Array s A_1 A_2 A_3 e s B_1 B_2 B_3 e s C_1 C_2 C_3 e End ------------------------------------------------------------------------------- Columns of Data pr To convert a file (or part of a file) to columns. Warning: this program will truncate data to fit into space available! Example file: numbers 1 - 10, one number per line It is recomended that the -# be the first option. Down the columns (like ls) pr -3 -t -l3 | `-- number of columns `----- lines per column (very important to give correctly) if output width is a problem add a line width -w80 EG: seq -f %02g 10 | pr -3 -t -l3 -w30 Outputs: 01 04 07 02 05 08 03 06 09 10 paste Across paste - - - Number of - given determines number of columns (tab separated) EG: seq -f %02g 10 | paste - - - Outputs: 01 02 03 04 05 06 07 08 09 10 OR (with posible data truncation to fit ouptput line width) you can use "pr" with one-line 'pages'. pr -t -l1 -3 xargs (unformated 3 words per line) seq 13 | xargs -n 3 echo Outputs: 1 2 3 4 5 6 7 8 9 10 11 12 13 Feed to column -t to re-align columns EG: seq 13 | xargs -n 3 echo | column -t Outputs: 1 2 3 4 5 6 7 8 9 10 11 12 13 If you need the collected filenames in the middle of a command.. seq 13 | xargs -n 3 sh -c 'echo -- "$@" --;' a -- 1 2 3 -- -- 4 5 6 -- -- 7 8 9 -- -- 10 11 12 -- -- 13 -- column On linux to just put the data into columns use "column" But you don't get good control of the number of columns. Example: seq -f %03g 5 5 500 | column Or on most linux machines.. column -x -c30 `--- character width of output line This can also format text tables using "-t" function of column to preserve the line by line structure of the input file. column -s: -t /etc/group OR a more complex example... ( echo "PERM LINKS OWNER GROUP SIZE MONTH DAY HH:MM NAME"; \ ls -l | tail -n+2; \ ) | column -t Warnings: * It also does not work as well as should EG: compare the output of "ls" and "ls | column" * No easy contol of number of columns * Tabs are used. Use the "expand" filter to remove them before using the "column" filter. * Any meta-characters found may cause column silently ABORT, truncating the file being processed! perl You can also using a non-standard perl module (from CPAN) seq -f %03g 100 |\ perl -MArray::PrintCols -e '@a=<>; chomp @a; print_cols \@a' Otherwise manually handle columns in perl (read all in first) seq -f %03g 100 |\ perl -e ' @a=<>; chomp @a; my ( $c, $i ) = (10,10); foreach ( @a ) { print("\n"),$i=$c unless $i; printf " %4s", $_; $i--; } print "\n"; ' cut Extract columns from a file in terms of charcater separated fields or by character position. colrm Remove unwanted columns of text from the file, in terms of character positions. Note "cut" extracts just teh column, "colrm" returns all but the column specified. That is they are the inverse of each other. ------------------------------------------------------------------------------- Suppressing the shell background fork message (*csh) The trick is to redirect the standard error output of the _shell_ itself Korn (with loss of standard error) ( command & 2>&3 ) 3>&2 2>/dev/null csh/tcsh (this appears to work) ( command & ) bourne shell does not give a message about background forks ever. The real problem is doing this without loosing the stderr of the command BASH This works. ( command & ) But on older BASH you can use this... ( command 2>&5 & ) 5>&2 2>/dev/null For example... This produces extra forking JUNK.. ( echo "stdout"; echo "stderr" >&2; ) & [1] 29886 stdout stderr [1]+ Done ( echo "stdout"; echo "stderr" 1>&2 ) This does not ( ( echo "stdout"; echo "stderr" >&2; ) & ) stdout stderr ------------------------------------------------------------------------------- Getting environment from CSH startup scripts... env - DISPLAY=$DISPLAY HOME=$HOME TERM=dumb \ csh -cf 'set prompt="> "; source .cshrc; source .login; echo "#------ENVIRONMENT------" env' | sed -n '/#------ENVIRONMENT------/,$p' ------------------------------------------------------------------------------- Am I a Non-Interactive Shell bourne sh: if [ -z "$PS1" ]; then # script/remote execution (non-interactive) csh: if ( ! $?prompt ) then # script/remote execution (non-interactive) In the bourne shell a better way is to test the shell options ``$-'' for the interactive flag directly. case $- in *i*) ;; # do things for interactive shell *) ;; # do things for non-interactive shell esac ------------------------------------------------------------------------------- Auto Background a shell script #!/bin/csh -f if ( $1:q != '...' ) then ( $0 '...' $*:q & ) exit 0 endif shift ...rest of script to run in background... OR #!/bin/sh foreground stuff ( ( background stuff ) & ) exit ------------------------------------------------------------------------------- Background Remote Processes A remote command leaves lots of `extra' processes to hold a communications channel open for IO. The following will background the remote shell and completely close the IO channel of the rsh command. Csh on remote machine rsh machine -n 'command >&/dev/null /dev/null 2>&1 <&1 &' Either Shell or other shell rsh machine -n '/bin/sh -c "exec command >/dev/null 2>&1 <&1" &' ------------------------------------------------------------------------------- Loop until parent process dies Background tasks have an annoying habit of continuing to run AFTER you have logged out. This following example program looks up the launching parent process (typically your login shell) and only loops if the parent process is still alive. WARNING: The PS command varies from UNIX system to UNIX system so you will have to tweek the arguments to the `ps' command to make this script work on your UNIX system =======8<-------- #!/bin/sh # # Loop until parent dies # sleep_time=300 # time between background checks # Pick the appropriate ps options for your UNIX system # Uncomment ONE of the following lines #job_opt=xj; sep=""; ppid=1; # SunOS job_opt=xl; sep=" "; ppid=4; # Solaris #job_opt=xl; sep=" "; ppid=4; # IBM UNIX (aix) #job_opt=xl; sep=" "; ppid=4; # SGI UNIX (irix) # Discover the parents process ID set - `ps $job_opt$sep$$ | tail -n+2` "1" eval parent=\$$ppid # While parent is still alive # The kill command "-0" option checks to see if process is alive! # It does NOT actually kill the process (EG: test process) while kill -0 $parent 2>/dev/null; do # ... # Do the background job here # ... sleep $sleep_time done # Parent process has died so we also better die. exit 0 =======8<-------- Also see the script ~anthony/bin/scripts/hostspace as an example of a shell script for multiple hosts ------------------------------------------------------------------------------- Setting a timed alarm in shell This can be done (as a background task) exactly how is an another matter As an educated guess probably something like.. # Trap the USR1 signal trap "do timeout commands" 16 ( sleep $timeout; kill -16 $$; ) & See also "Command Timeout" (next). ------------------------------------------------------------------------------- Command Timeout Run a command but kill it if it runs for too long. This prevents scripts hanging on a command, especially network related commands, like nslookup. Simple C solution... Look at ~/store/c/programs/timeout.c which will timeout a command simply and easilly. A lot better than the complex solutions below. But of course it is binary, so does not work on all systems. A advanced version is on linux systems in /usr/bin/timeout There is a "timeout" program written in shell. ~/store/scripts/command_run/timeout_bad It does a looped sleep to do interval testing, as such may not instantly return when a command finishes. Also while it works fine from the command line it fails when used within a shell script! The following are my attempts to build it in shell! And shows just what a pain it can be. ---- Attempt 1 ---- Simply runs the command in the background and waits for it to complete. A sleep command provides a timeout. Works but full timeout period is always waited before exiting TIMEOUT=60 # timelimit for command ( command_which_can_hang & sleep $TIMEOUT; kill $! 2>/dev/null ) The above will wait for the sleep for the full TIMEOUT period, regardless of how fast the command completes. If the command is known to never finish then the above will be fine! For example.. at 3 in the afternoon, play random noise for 10 seconds The 'cat' normally would never end, so will never abort early. The sleep will time out the never ending command. > at 3pm cat /dev/urandom > /dev/dsp & sleep 10; kill $! As a test try this... sleep 1 cat /dev/urandom > /dev/dsp & sleep 1; kill $! ---- Attempt 2 ----- Works for modern day shells... Example: NFS quota can hang, (so can any network command!) =======8<-------- QUOTA_TIMEOUT=5 do_quota() { # lookup the users disk quota but with a timeout quota -v "$@" & # output to stdout command_pid=$! # get its pid (to terminate) ( trap "exit 1" 1 2 3 15 # be quiet if terminated sleep $QUOTA_TIMEOUT echo >&2 "Quota Timeout" kill -9 $command_pid exit 0 ) & timeout_pid=$! # pid of the timeout process wait $command_pid status=$? kill $timeout_pid exit $status } quota=`do_quota $USER` =======8<-------- The problem here is that this works fine but the sleep will still continue for the full timeout period. Basically the parent has no simple way of killing the sleep that was lanched in the sub-shell, and does nto die naturally. This is not a problem if the sleep is not too long, it does not use any resources other than a entry in the process table. ---- Attempt 3 ---- Kill the timer sleep too! This is amost exactly the same, but also ensures the timer sub-process cleans up the sleep if it is still running. =======8<-------- DNS_TIMEOUT=5 do_nslookup() { # lookup the users disk quota but with a timeout nslookup "$@" | sed -n 's/Address: *\(.*\)/\1/p' & command_pid=$! # get its pid (to terminate) # run a timer to abort the above command ( trap 'kill -ABRT $sleep_pid; exit 1' 1 2 3 15 sleep $DNS_TIMEOUT & sleep_pid=$! wait $sleep_pid kill -9 $command_pid exit 1 ) & timeout_pid=$! # pid of the timeout process # Wait for command, or timeout wait $command_pid status=$? # cleanup timer (if still running) kill $timeout_pid 2>/dev/null wait $timeout_pid 2>/dev/null exit $status } IP=`do_nslookup google.com` echo "Google is at IP address: $IP" =======8<-------- The last method has now been built into a program http://www.ict.griffith.edu.au/anthony/software/#timeout Which includes linkes to a simple test progrom called "countdown" It can be tested with timeout 5 countdown timeout 12 countdown ------------------------------------------------------------------------------- Protecting shell scripts from ^Z This signal can't normally be stopped in a shell, the trick is to change key generating the signal (Don't forget to return it to normal). stty susp undef -- if available stty susp '^-' -- maybe system dependant ------------------------------------------------------------------------------- Progress Bar pv or "pipeview" When reading from a fixed file the 'pv' or "pipe viewer" command tells you how much progress has been made. pv somefile | gzip > rt94-171-06.gz 128MB 0:00:15 [ 9.1MB/s] [=====>.....................] 18% ETA 0:01:07 When taring up a directory getting disk size first is needed tar -czf - . | pv -s $(du -sb | grep -o '[0-9]*') > out.tgz 44.3MB 0:00:27 [1.73MB/s] [>..........................] 0% ETA 13:36:22 If size is not known, it just shows 'activity' indicator, and how much has been read, for how long, and average speed of data transfer pv /dev/urandom > /dev/null 38.2MB 0:00:03 [12.8MB/s] [ <=> ] This is a all bash method (except for the sleep) bar="==================================================" barlength=${#bar} i=0 while ((i < 100)); do n=$((i*barlength / 100)) # Number of bar segments to draw $((i/2)) printf "\r[%-${barlength}s]" "${bar:0:n}" ((i += RANDOM%5+2)) # i = percentage done sleep 1 done echo Of course 'i' should be set from somewhere meaningful. A perl method with time: spent, left, and total; I think is better. ------------------------------------------------------------------------------- Random number generation in a shell nawk: set range = ???? set random = `nawk 'BEGIN { srand(); printf "%d", rand()*'$range' }' /dev/null` date: set range = ???? # file length -- `wc -l <$file` set date = `date +%j%H%M%S` set random = `expr $date \% $range` (k/z/ba)sh $RANDOM Posible improvment... !/bin/bash MX="0123456789" NL="5" # size of the random number wanted while [ ${n:=1} -le $NL ] do NUM="$NUM${MX:$(($RANDOM%${#MX})):1}" let n+=1 done echo "$NUM" Programed (csh) (could be via expr too) set multiplier = 25173 set modulus = 65536 set increment = 13849 set seedfile = $HOME/.rnd # seed file to use if ( ! -f $seedfile ) then echo '17' > $seedfile endif @ number = ( `cat $seedfile` * $multiplier + $increment ) % $modulus echo $number > $seedfile @ number = $number % $range echo $number ------------------------------------------------------------------------------- increment a character in shell char=`echo $char | tr ' -~' '\!-~'` or (sutable for hex chars) str1="ABCDEFGHIJKLMNOPQRSTUVWXYZ" str2="BCDEFGHIJKLMNOPQRSTUVWXYZA" pos=`expr index $str1 $char` char=`expr substr $str2 $pos 1` echo $char In perl the "increment alpha string" makes this easy char=abzzz char=`perl -e '$n="'"$char"'"; print ++$n'` echo $char acaaa ------------------------------------------------------------------------------- Date - Four Digit Year in shell (for logfiles) For Newer UNIX Machines, IEEE date format date=`date +'%Y-%m-%d %R:%S'` For Older UNIX machines (SunOS)... While the newest UNIX's allow a %Y for a four digit year older machines like SunOS does NOT. This means the year will need to be extracted from the date output (or alternative format code). year=`date | sed 's/.*\([0-9][0-9][0-9][0-9]\).*/\1/'` date=`date $year-%m-%d %H:%M:%S" If you have the MH mail system install you can convert the date to rfc822 date format, use the "dp" program. This has its own mh output format codes. /usr/lib/mh/dp "`date`" or /usr/lib/mh/dp -format '%(year{text})' "`date`" or /usr/lib/mh/dp \ -format '%04(year{text})-%02(mon{text})-%02(mday{text})' \ "`date`" ------------------------------------------------------------------------------- Date Sorting Sorting logs with time like... [03/07/2013 08:18:45 AM] PikaPika01 logged in. [03/07/2013 08:19:55 AM (world: 975,86,2494)] PikaPika01: hey This has problems... 1/ The date order is american MM/DD/YYYY format 2/ AM and PM must be sorted before the actual time field 3/ "AM]" and "AM (.." words sort differently (ignore trailing chars) 4/ "12" is before "01" that is 12 AM is before 01 AM Here is my solution... cat file | sed 's/ 12:/ 00:/' | sort -t\ -k1.8,1.11 -k1.2,1.3 -k1.4,1.5 -k3,3.2 -k2,2 -k4 | sed 's/ 00:/ 12:/' >merged.log Note the '12' is modified to '00' for the sort, then converted back ------------------------------------------------------------------------------- Date - Time in Seconds, NOW! (for later comparision) EG: getting the number of seconds since epoch (midnight, 1/1/1970, GMT) This is for the purposes of timing various command and comparing the start and end times. The problem is as there is no standard way for geting the system clock under normal plain shell scripts. Under linux (Gnu-date) you can date +%s With perl you can use... perl -e 'print time(), "\n"' Seconds stince year start (for short time deltas)... Do not rely on it for time deltas (how long) as it does not handle years (which is a variable time period). NOTE: %j is day within the year date +'%j * 86400 + %H * 3600 + %M * 60 + %S' | bc Seconds back to Date.... BASH... bash -c "printf '%(%Y-%m-%d %R:%S)T\n' 1000000000" PERL... perl -e 'require "ctime.pl"; print &ctime(1000000000); (for more see the perl/general.hints page) ------------------------------------------------------------------------------- Date of Yesterday (without a lot of fuss) Getting yesterdays day is usually best done in either C or perl. #!/usr/bin/perl require "ctime.pl"; print &ctime($^T - 60*60*24); alternativally perl -MPOSIX -le '@now = localtime; $now[3] -= 1; print scalar localtime mktime @now' By changing your timezone you can fake the getting of yesterdays date, But ONLY if you live in Australia where it is already tomorrow!. It is not perfect (2 hours short but close enough in most cases) env TZ=GMT-12 date under linux (GNU date) this is now very easy date -d yesterday date -d 'last week' date -d '-4 hours' date -d '1 month ago' date -d "today -1 days" +%Y%m%d or from a specific date date -d "2010-01-01 +10 days" +"# %Y-%m-%d" # 2010-01-11 seq 30 | xargs -n 1 -i{} date -d "2010-01-01 +{} days" +"# %Y-%m-%d" ------------------------------------------------------------------------------- Merge multiple blank lines into one line. This is not easy, as we want to preserve a blank line, removing extras. Two basic techniques: print paragraphs, or delete extra blanks. awk '{ printf "%s ", $0 } NF == 0 { print "\n" }' filename cat -s # will do this (if available) perl -ne 'if (/\S/) { print; $i=0 } else {print unless $i; $i=1; }' perl -ne 'print if /\S/../^\s*$/' sed '/./,/^$/!d' # no blank lines at to one at bottom sed '/^$/N;/\n$/D' # one black at top, no blanks at bottom sed '/[^ ]/,/^[ ]*$/!d' # For blank lines with spaces and tabs # Line joining in the vim editor! as a macro :map QE :$s/$/\\rZ/:g/^[ ]*$/,/[^ ]/-jGdd or :%s/\n\s*\n\(\s*\n\)*/\r\r/ ------------------------------------------------------------------------------- Multi-line Paragraph to Single-line Paragraph Most text files including this one consists of paragraphs of multiple lines seperated by a blank line. Convert all paragraphs into a single line (blank line seperated) sed '/^$/d; :loop y/\n/ /; N; /\n$/! b loop; s/ */ /g; s/^ //; s/ $//' Also remove blank lines between paragraph lines sed '/^$/d; :loop N; s/\n$//; T loop; y/\n/ /; s/ */ /g; s/^ //; s/ $//' WARNING: better space handling is probably needed, especially for a last paragraph that has no final blank line after it. ------------------------------------------------------------------------------- One word per line This is typically used for pre-processing for text formatting (see next) or as part of a word-diff type utility (see: "wdiff", "sim_text" utilities). cat file | sed 's/[^[:alnum:][:space:]]/\\&/g' | xargs -rn1 echo The 'sed' in the above is to quote non-alphabetrical characters. To remove punctuation too, delete the '\\&' form the above Note paragraph blank lines are also removed. ASIDE: Removing '[:space:]' is the accepted way of handling return delimited line input into "xargs" safely. The better way is to use -0 filename listing format. ------------------------------------------------------------------------------- Word-Wrapping or Text Formatting Single-line to Multi-line Paragraphs Usually you would use "fmt" or "fold" which is a standard addition for LINUX systems. fmt -u -w 72 NOTE: fmt preserves the indentation of the lines... fold -s -w 72 [file...] NOTE: fold does not word-wrap multi-line paragraphs! Here is a 'sed' command to remove all newlines not involved with a blank line. expand < file | sed '/^$/d; :loop y/\n/ /; N; /\n$/! b loop; s/ */ /g; s/^ //' | fold -s -w 72 This replaces fold with a poor mans word-wrap expand < file | sed '/^$/d; :loop y/\n/ /; N; /\n$/! b loop; s/ */ /g; s/^ //' | sed 's/ $/@/; s/[^@]\{55\} /&@/g' | tr '@' '\012' | sed 's/^/ /' Note the second sed command wraps on a space AFTER '55' characters. As such it can be fooled by very very long words. It should wrap on the space before the given column limit. In "gnu-sed" you can replace '@' with '\x01' and in "tr" use '\001' This first joins the text into one long line, then inserts a marker for word-wrapping points. Before breaking AFTER 55 characters. Finally it indents the whole paragraph. A variation for handling multiple blank line seperated paragraphs is to first remove all extra spaces including those at the start and end of line, then treat any two spaces as paragraph break. expand < file | sed 's/ */ /g; s/^ //; s/ $//;' | tr '\012' ' ' | sed 's/ $/@/; s/ */@@/g; s/[^@]\{55\} /&@/g' | tr '@' '\012' Using xargs! The number is column limit + 5 for the "echo " NOTE this only handles a single paragraph. expand < file | sed 's/[^[:alnum:][:space:]]/\\&/g' | xargs -r -s 85 echo Perl Text Format (using Text::Wrap) #!/usr/bin/perl use Text::Wrap; $Text::Wrap::columns = 78; $Text::Wrap::huge = 'overflow'; $_ = 'words' print(Text::Wrap::wrap(' ', ' ', $_),"\n") vim has built in formating. gg}gqGgg} ------------------------------------------------------------------------------- Capitalize the first word. The initial solutions in the news group cam out to more than 10 lines of code! # Ken Manheimer (expr-tr-expr) -- see Expr Warning Word=`expr + "$word" : "\(.\).*" | tr a-z A-Z``expr + "$word" : ".\(.*\)"` # Paul Falstad (cut-tr-sed) Word=`echo $word|tr a-z A-Z|cut -c1``echo $word|sed s/.//` ==> # Logan Shaw (cut-tr-cut) Word=`echo "$word" | cut -c1 | tr [a-z] [A-Z]``echo "$word" | cut -c2-` # Harald Eikrem (sed only) Word=`echo $word | sed -e ' h; 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/; G; 's/\(.\).*\n./\1/; ' ` # Tom Christiansen (perl) Word=`echo $word | perl -pe 's/.*/\u$&/'` # Jean-Michel Chenais # (korn shell built-ins only) typeset -u W;typeset -l w;W=${word%${word#?}};w=$W;Word=$W${word#${w}} # perl of course makes this easy perl -e 'print "\u'"$word"'"' ------------------------------------------------------------------------------- Argument Replacement Wrapper (BASH) This function replaces the "convert" command with a variation the replaces specific options, such thouse caused by version changes. convert() { # go through arguments and replace "-rotate" with "+distort" "SRT" args=() # empty array for arg in "$@"; do case "$arg" in "-matte") args=( "${args[@]}" "-alpha" "set" ) ;; # replace "-matte" "+matte") args=( "${args[@]}" "-alpha" "off" ) ;; # replace "+matte" *) args=( "${args[@]}" "$arg" ) ;; # just copy argument as is esac done # call the the REAL command (with debugging output) echo "DEBUG: convert ${args[*]}" command convert "${args[@]}" } convert rose: -matte -background none -rotate 40 show: The above also allows you to add things like * path to real command to execute, or call a different command * prefix extra setup options or other hidden options * create 'fake' options that expand (macro like) to other options * call other programs to pre-determine the options to use WARNING: Watch out that for arguments that look like options... For example convert rose: -gravity center -annotate 0 "-matte" show: Here "-matte" is a string argument, and not a option! Used as is with the above example function will results in convert rose: -gravity center -annotate 0 -alpha set show: which will annotate the string "-alpha" instead of "-matte", and generating the warning: "unable to open image 'set'" --- The following is a more complex version that will take care options that could take multiple arguements, and keep such arguments separate. However this is really getting too difficult as you need to start looking at very option that has arguments! convert() { # go through arguments and replace "-rotate" with "+distort" "SRT" args=() # the arguments for the real command option=() # option and arguments (may have multiple arguments) count=0 # number of arguments to still add to option for arg in "$@"; do # collect arguments into multi-word options if [ $count -eq 0 ]; then option=() option="$arg" # note that this is an option case "$option" in "-annotate") count=2 ;; # annotate has two arguments "-rotate") count=1 ;; # rotate has one argument esac else # read in any more option arguments needed option=( "${option[@]}" "$arg" ) count=$( expr $count - 1 ) fi [ $count -gt 0 ] && continue # Substitute options with replacements case "$option" in "-matte") option=( "-alpha" "set" ) ;; "+matte") option=( "-alpha" "off" ) ;; "-rotate") option=( "-virtual-pixel" "background" "+distort" "SRT" "${option[1]}" ) ;; esac args=( "${args[@]}" "${option[@]}" ) # just copy option as is done # call the command - printing what will actually be executed echo "DEBUG: magick ${args[*]}" command magick "${args[@]}" } For example this will now work... convert rose: -gravity center -annotate 0 "-matte" \ -matte -background none -rotate 40 show: ------------------------------------------------------------------------------- Curses in shell script To use termcap entries in a shell script use the `tput' command EXAMPLES /usr/5bin/tput bold # bold (extra half brigtht mode) /usr/5bin/tput bink # bink mode (if available) /usr/5bin/tput rev # reverse video /usr/5bin/tput smul # underline mode /usr/5bin/tput smso # standout (usually reverse) /usr/5bin/tput rmso # end standout (return to normal) /usr/5bin/tput clear # clear screen /usr/5bin/tput cup 5 23 # cursor to move to row 5 column 23 See terminfo(5) for more info. ------------------------------------------------------------------------------- Split pipe into two separate commands (executable tee) awk '{ print | "'cmd1'" ; print }' | cmd 2 See also the 'tpipe' command. ------------------------------------------------------------------------------- Find a Process of a particular name --- see csh alias pf Using grep (but ignore all grep processes) ps auxgww | grep "$NAME" | grep -v grep | cut -c1-15,36-99 Merging the two greps.. ps uxw | grep "[s]sh-agent" | awk '{print $2}' The "[s]sh-agent" will not match the grep process itself! EG: it will not match the "[s]sh-agent" string in the grep process Using awk (but ignoring all awk processes)... ps auxgww | awk "/$NAME/ && \! /(awk)/" | cut -c1-15,36-99 or for a exact process name ps auxgww | awk '/(^| |\(|\/)$NAME( |\)|$)/' | cut -c1-15,36-99 or alturnativeally which matches under a lot of conditions... EG: matches :01 NAME arg :01 some/path/NAME :01 NAME: ps auxgww | \ awk '/:[0-9][0-9] (|[^ ]*\/)$NAME($| |:)/' | cut -c1-15,36-99 Perl version (also matches on username) ps auxgww | \ perl -nle 'print if $. == 1 \ || /^\s*\!:1\s/o \ || /:\d\d (|\[ *|[^ ]*\/)\!:1($|[]: ])/o; ' As you can see things can get complex very quickly ------------------------------------------------------------------------------- Context Grep Or how to display lines before/after search pattern. GNU grep, has context built in to it, check out the -A, -B, and -C options. No gnu-grep grep -v pattern file | diff -c3 - file | grep '^. ' | colrm 1 2 or grep -n $1 $2 | awk -F: '{ print $1 }' | while read linenum do awk 'NR>target-5 && NR/dev/null || echo {} >> url_failed " #or cat url_list | xargs -n 1 -P 8 "wget -q {} 2>/dev/null || echo {} >> url_failed " -------------------------------------------------------------------------------