Python Concepts/Interfacing with Unix
Objective
edit
|
Lesson
edit
Invoking python from UnixeditTo enter python in interactive mode: $ /usr/local/bin/python3.6
Python 3.6.3 (v3.6.3:2c5fed86e0, Oct 3 2017, 00:32:08)
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> 3+4
7
>>> quit()
$
If your $PATH global variable contains the directory /usr/local/bin, you can invoke python with a file name rather than a path name: $ echo $PATH
/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin
$ python3.6
Python 3.6.3 (v3.6.3:2c5fed86e0, Oct 3 2017, 00:32:08)
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> 5 * 7
35
>>> quit()
$
Let test.py be the name of a python script. $ cat test.py
# test.py
print ('3+4 =',3+4)
$
$ python3.6 test.py
3+4 = 7
$
$ cat test.py | python3.6
3+4 = 7
$
$ python3.6 < test.py
3+4 = 7
$
$ echo print\(\'3\+4\ \=\'\,3\+4\) | python3.6
3+4 = 7
$
$ echo "print ('3+4 =',3+4)" | python3.6
3+4 = 7
$
$ echo python3.6 ./test.py | ksh
3+4 = 7
$ echo python3.6 ./test.py | csh
3+4 = 7
$
Let the first line of your script contain the path name of python with appropriate syntax: $ cat test.py
#!/usr/local/bin/python3.6
# test.py
print ('11-5 =',11-5)
$
$ ls -la test.py
-rwxr--r-- 1 name staff 62 Nov 8 16:13 test.py # Ensure that the script is readable and executable.
$
$ ./test.py # The script can be executed as a stand-alone executable.
11-5 = 6
$
Passing arguments to pythonedit$ cat test.py
#!/usr/local/bin/python3.6
# test.py
import sys
print ('''
sys.argv[1] = "{}"
sys.argv[2] = "{}"
sys.argv[3] = "{}"
Number of arguments received = {}
'''.format(
sys.argv[1] ,
sys.argv[2] ,
sys.argv[3] ,
len( sys.argv[1:] )
))
$
$ ./test.py arg_1 arg\ 2 'arg 3' "arg 4" arg\*5
sys.argv[1] = "arg_1"
sys.argv[2] = "arg 2"
sys.argv[3] = "arg 3"
Number of arguments received = 5
$
Invoking Unix from pythoneditThis section uses function
>>> import subprocess
>>>
>>> a = subprocess.run('date') ; a
Sat Nov 9 16:57:51 GMT 2019
CompletedProcess(args='date', returncode=0)
>>> a.args
'date'
>>> a.returncode
0
>>>
>>> type(a)
<class 'subprocess.CompletedProcess'>
>>>
The >>> a = subprocess.run('date',stdout=subprocess.PIPE) ; a
CompletedProcess(args='date', returncode=0, stdout=b'Sat Nov 9 17:35:36 CST 2019\n')
>>> a.args
'date'
>>> a.returncode
0
>>> a.stdout
b'Sat Nov 9 17:35:36 UST 2019\n'
>>> a.stdout.decode()
'Sat Nov 9 17:35:36 UST 2019\n'
>>>
To execute a Unix command with arguments: >>> a = subprocess.run(('ls','-laid','.'),stdout=subprocess.PIPE) ; a
CompletedProcess(args=('ls', '-laid', '.'), returncode=0, stdout=b'27647146 drwxr-xr-x 22 name staff 748 Nov 8 16:33 .\n')
>>> a.args
('ls', '-laid', '.')
>>> a.returncode
0
>>> a.stdout.decode()
'27647146 drwxr-xr-x 22 name staff 748 Nov 8 16:33 .\n'
>>>
Notice the difference between the next two commands: >>> a = subprocess.run(( 'echo', 'print()', '|', 'python3.6' ),stdout=subprocess.PIPE) ; a
CompletedProcess(args=('echo', 'print()', '|', 'python3.6'), returncode=0, stdout=b'print() | python3.6\n')
>>>
>>> a.args
('echo', 'print()', '|', 'python3.6') # 4 arguments.
>>> a.stdout
b'print() | python3.6\n'
>>>
>>> a = subprocess.run( 'echo "print(2*7)" | python3.6',stdout=subprocess.PIPE,shell=True) ; a
CompletedProcess(args='echo "print(2*7)" | python3.6', returncode=0, stdout=b'14\n')
>>>
>>> a.args
'echo "print(2*7)" | python3.6' # 1 argument.
>>> a.stdout
b'14\n' # This is probably what you wanted.
>>>
Sending data to a pipeeditThis process depends on the Unix command $ echo echo -n | ksh
$ echo echo -n | csh
$ echo echo -n | bash
$ echo echo -n | sh
-n
$
Python executable #!/usr/local/bin/python3.6
# pecho.py
import sys
Length = len( sys.argv[1:] )
if Length not in (1,2) :
print ('pecho.py: Length {} not in (1,2).'.format(Length), file=sys.stderr)
exit (39)
if Length == 2:
if sys.argv[1] not in ('-n','-N') :
print ('pecho.py: expecting "-n" for sys.argv[1]', file=sys.stderr)
exit (38)
print (sys.argv[2], end='') ; exit(0)
if sys.argv[1] in ('-n','-N') :
print ('pecho.py: received only "-n"', file=sys.stderr)
exit (37)
print (sys.argv[1])
The perl script below will be formatted and sent to a pipe for execution by perl. import re
import subprocess
perlScript = r'''
# This is a 'short' perl script.
$a = 5 ;
$b = 8 ;
$newLine = "
";
(length($newLine) == 1) or die '$newLine'." must not contain extraneous white space.";
print ('a+b=',$a+$b,$newLine) ;
print ("b-a=",$b-$a,"\n") ;
'''
repl = """'"'"'"""
pS1 = re.sub( "'", repl, perlScript )
pS1 = "'" + pS1 + "'"
a = subprocess.run( "./pecho.py -n " + pS1 + " | perl" ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE, shell=True)
print ('''
a.args =
{}
a.returncode = {}
a.stdout = {}
{}]]]]
'''.format(
a.args,
a.returncode ,
a.stdout ,
a.stdout.decode(),
),end='')
a.args =
./pecho.py -n '
# This is a '"'"'short'"'"' perl script.
$a = 5 ;
$b = 8 ;
$newLine = "
";
(length($newLine) == 1) or die '"'"'$newLine'"'"'." must not contain extraneous white space.";
print ('"'"'a+b='"'"',$a+$b,$newLine) ;
print ("b-a=",$b-$a,"\n") ;
' | perl
a.returncode = 0
a.stdout = b'a+b=13\nb-a=3\n'
a+b=13
b-a=3
]]]]
Processing errorseditIf you execute command import subprocess
import sys
a = error = ''
try : a = subprocess.run(.........)
except : error = sys.exc_info()
# You expect exactly one of (a, error), preferably a, to be True.
set1 = {bool(v) for v in (a, error)}
if len(set1) != 2 : print ('Internal error.') ; exit (99)
TimeoutExpirededittry : a = subprocess.run(('sleep', '100'), stdout=subprocess.PIPE, timeout=2 )
except subprocess.TimeoutExpired:
print ('Detected timeout.')
t1 = sys.exc_info()
print ('sys.exc_info() =',t1)
isinstance(t1,tuple) or exit(99)
len(t1)==3 or exit(98)
isinstance(t1[1], subprocess.TimeoutExpired) or exit(97)
print('''
cmd = {}
timeout = {}
stdout = {}
stderr = {}
'''.format(
t1[1].cmd,
t1[1].timeout,
t1[1].stdout,
t1[1].stderr,
))
Detected timeout.
sys.exc_info() = (<class 'subprocess.TimeoutExpired'>, TimeoutExpired(('sleep', '100'), 2), <traceback object at 0x10186c988>)
cmd = ('sleep', '100')
timeout = 2
stdout = b''
stderr = None
CalledProcessErroreditIf check is true, and the process exits with a non-zero exit code, a CalledProcessError exception will be raised. try : a = subprocess.run('cat crazy_file_name', shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
except subprocess.CalledProcessError :
error = sys.exc_info()
print ('''
CalledProcessError detected.
cmd = {}
returncode = {}
stdout = {}
stderr = {}
'''.format(
error[1].cmd,
error[1].returncode,
error[1].stdout,
error[1].stderr
))
CalledProcessError detected.
cmd = cat crazy_file_name
returncode = 1
stdout = b''
stderr = b'cat: crazy_file_name: No such file or directory\n'
Normal processing of errorseditIf you're not using the a=error=''
try : a = subprocess.run(('cat', 'crazy_file_name'), stdout=subprocess.PIPE, stderr=subprocess.PIPE )
except : error = sys.exc_info()
set1 = {bool(v) for v in (a, error)}
if len(set1) != 2 : print ('Internal error.') ; exit (99)
if a :
print ('a =',a)
print ('''
args = {}
returncode = {}
stdout = {}
stderr = {}
'''.format(
a.args ,
a.returncode ,
a.stdout ,
a.stderr ,
))
a = CompletedProcess(args=('cat', 'crazy_file_name'), returncode=1, stdout=b'', stderr=b'cat: crazy_file_name: No such file or directory\n')
args = ('cat', 'crazy_file_name')
returncode = 1
stdout = b''
stderr = b'cat: crazy_file_name: No such file or directory\n'
In the above invocation of subprocess.run(...), the exception was not taken and all information about the error is available in a.returncode and a.stderr. With a few well chosen Unix commands, subprocess.run gives you easy access to much good information about the Unix environment. # Does the file ../ObjOrPr/test.py exist?
>>> try : subprocess.run('head -1 test.py', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,cwd='../ObjOrPr' )
... except : sys.exc_info()
...
CompletedProcess(args='head -1 test.py', returncode=0, stdout=b"print ('4 + 9 =',4+9)\n", stderr=b'')
# Yes, and it's readable.
>>> try : subprocess.run('cat /dev/null >> test.py', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,cwd='../ObjOrPr' )
... except : sys.exc_info()
...
CompletedProcess(args='cat /dev/null >> test.py', returncode=1, stdout=b'', stderr=b'/bin/sh: test.py: Permission denied\n')
>>>
# But it's not writeable.
try and except statements provide good clean info about errors: >>> try : subprocess.run('pwd', stdout=subprocess.PIPE, stderr=subprocess.PIPE,cwd='../OjOrPr' )
... except : sys.exc_info()
...
(<class 'FileNotFoundError'>, FileNotFoundError(2, "No such file or directory: '../OjOrPr'"), <traceback object at 0x10199e2c8>)
>>>
Without try and except: >>> subprocess.run('pwd', stdout=subprocess.PIPE, stderr=subprocess.PIPE,cwd='../OjOrPr' )
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 403, in run
with Popen(*popenargs, **kwargs) as process:
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 709, in __init__
restore_signals, start_new_session)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 1344, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: '../OjOrPr': '../OjOrPr'
>>>
|
Addendum
edit
Default value shell=FalseeditThe reference recommends default value >>> a = subprocess.run( ('ls','-laid','.','..',) ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE)
>>> print(a.stdout.decode())
27647146 drwxr-xr-x 28 name staff 952 Nov 16 14:45 .
26949099 drwxrwxrwx 170 name staff 5780 Nov 8 14:36 ..
>>>
>>> a = subprocess.run( ('echo','Te$t p9tte8n:~!@:$%^&*()_+') ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE)
>>> print(a.stdout.decode())
Te$t p9tte8n:~!@:$%^&*()_+
>>>
However: >>> a = subprocess.run( ('echo','Te$t p9tte8n:~!@:$%^&*()_+', '|', 'od', '-h') ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE)
>>> print (a.returncode)
0
>>> print(a.stdout.decode())
Te$t p9tte8n:~!@:$%^&*()_+ | od -h # This probably is not what you wanted.
>>>
Try again with >>> a = subprocess.run( ('echo "Te$t p9tte8n:~!@:$abcde%^&*()_+" | cat') ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE,shell=True)
>>> print(a.stdout.decode())
Te p9tte8n:~!@:%^&*()_+ # '$t' and '$abcde' are missing from output.
>>>
'$t' and '$abcde' are missing from output above because a Unix string beginning with '"' provides variable substitution. >>> a = subprocess.run( ("echo 'Te$t p9tte8n:~!@:$abcde%^&*()_+' | cat") ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE,shell=True)
>>> print(a.stdout.decode())
Te$t p9tte8n:~!@:$abcde%^&*()_+ # No variable substitution in string beginning with "'".
>>> a = subprocess.run( ("echo 'Te$t p9tte8n:~!@:$abcde%^&*()_+' | wc") ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE,shell=True)
>>> print(a.stdout.decode())
1 2 32 # 31 characters is correct length of pattern. "wc" adds new line.
>>>
Unix strings and Python stringseditProvided that the string to be passed to Unix does not contain a single quote "'", the simplest way to prepare the string is to enclose it in single quotes: >>> st1 = '\n line 1 \n line 2\n'
>>> print (st1, end='')
line 1
line 2
>>> st1a = "'" + st1 + "'"
>>>
>>> a = subprocess.run( ("echo " + st1a) ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE, shell=True)
>>> print (a.args)
echo '
line 1
line 2
'
>>> a.stdout.decode()[:-1] == st1
True
>>>
If the string contains a single quote, convert the single quote to a pattern which Unix will recognize. >>> st1 = '''ls -laid "Bill's file"''' # The string is: ls -laid "Bill's file"
>>> st1a = re.sub("'", """'"'"'""", st1)
>>> st1b = "'" + st1a + "'" # 'ls -laid "Bill'"'"'s file"' = 'ls -laid "Bill' + "'" + 's file"'
>>> a = subprocess.run( ("echo " + st1b) ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE, shell=True)
>>> print (a.args)
echo 'ls -laid "Bill'"'"'s file"'
>>> a.stdout.decode()[:-1] == st1
True
>>> a.stdout.decode()[:-1]
'ls -laid "Bill\'s file"'
>>>
>>>
>>> st1 = '''ls -laid "../ObjOrPr/Bill's file"'''
>>> st1a = re.sub("'", """'"'"'""", st1)
>>> st1b = "'" + st1a + "'"
>>> a = subprocess.run( ("echo " + st1b) ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE, shell=True)
>>> a.stdout.decode()[:-1]
'ls -laid "../ObjOrPr/Bill\'s file"'
>>> a.stdout.decode()[:-1] == st1
True
>>> a = subprocess.run( ("echo " + st1b + ' |ksh') ,stdout=subprocess.PIPE ,stderr=subprocess.PIPE, shell=True)
>>> print (a.args)
echo 'ls -laid "../ObjOrPr/Bill'"'"'s file"' |ksh
>>> print(a.stdout.decode(),end='')
27706761 -rw-r--r-- 1 name staff 0 Nov 18 16:22 ../ObjOrPr/Bill's file
>>>
To execute a shell scripteditimport re
import subprocess
shellScript = '''
cat ../ObjOrPr/"Bill's file" | wc
echo
ls -l ../ObjOrPr/Bill"'s f"ile
echo
ls -laid "../ObjOrPr/Bill's file" | od -c
'''
# Substiute 'Æ' for "'". This keeps the length the same and
# makes a.args slightly more readable.
sS1 = re.sub("'", 'Æ', shellScript)
sS2 = "'" + sS1 + "'"
pattern = "/Æ/s//'/g" ; p1 = '"' + pattern + '"'
a = subprocess.run( ("echo " + sS2 + ' | sed ' + p1 + ' | ksh') , stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
print ('''
a.args =
{}
a.returncode = {}
a.stdout =
{}]]]]
a.stderr =
{}]]]]
'''.format(
a.args,
a.returncode ,
a.stdout.decode(),
a.stderr.decode(),
),end='')
a.args =
echo '
cat ../ObjOrPr/"BillÆs file" | wc
echo
ls -l ../ObjOrPr/Bill"Æs f"ile
echo
ls -laid "../ObjOrPr/BillÆs file" | od -c
' | sed "/Æ/s//'/g" | ksh
a.returncode = 0
a.stdout =
0 0 0
-rw-r--r-- 1 _____Bill______ staff 0 Nov 18 16:22 ../ObjOrPr/Bill's file
0000000 2 7 7 0 6 7 6 1 - r w - r - -
0000020 r - - 1 _ _ _ _ _ B i l l
0000040 _ _ _ _ _ _ s t a f f 0
0000060 N o v 1 8 1 6 : 2 2 . .
0000100 / O b j O r P r / B i l l ' s
0000120 f i l e \n
0000125
]]]]
a.stderr =
]]]]
|
Assignments
editFurther Reading or Review
edit
|
References
edit
1. Python's documentation: 2. Python's methods: 3. Python's built-in functions:
|