Standard Library

Standard Library#

Note

Reference document for this section:

The Python Standard Library documentation: https://docs.python.org/3/library/index.html
Python Essential Reference, David Beazley, Addison-Wesley Professional

`os` module: operating system functionality#

“A portable way of using operating system dependent functionality.”

Directory and file manipulation#

Current directory:

import os
os.getcwd()

'/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language'

List a directory:

os.listdir(os.curdir)

['oop.md',
 'exceptions.md',
 'basic_types.md',
 'standard_library.md',
 'junk.txt',
 'python-logo.png',
 'demo2.py',
 'solutions',
 'workfile',
 '__pycache__',
 'test.py',
 'first_steps.md',
 'test.pkl',
 'functions.md',
 'my_file.py',
 'control_flow.md',
 'demo.py',
 'python_language.md',
 'data.txt',
 'io.md',
 'reusing_code.md']

Make a directory:

os.mkdir('junkdir')
'junkdir' in os.listdir(os.curdir)

True

Rename the directory:

os.rename('junkdir', 'foodir')
'junkdir' in os.listdir(os.curdir)

False

'foodir' in os.listdir(os.curdir)

True

os.rmdir('foodir')
'foodir' in os.listdir(os.curdir)

False

Delete a file:

fp = open('junk.txt', 'w')
fp.close()
'junk.txt' in os.listdir(os.curdir)

True

os.remove('junk.txt')
'junk.txt' in os.listdir(os.curdir)

False

`os.path`: path manipulations#

os.path provides common operations on pathnames.

fp = open('junk.txt', 'w')
fp.close()
a = os.path.abspath('junk.txt')
a

'/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/junk.txt'

os.path.split(a)

('/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language',
 'junk.txt')

os.path.dirname(a)

'/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language'

os.path.basename(a)

'junk.txt'

os.path.splitext(os.path.basename(a))

('junk', '.txt')

os.path.exists('junk.txt')

True

os.path.isfile('junk.txt')

True

os.path.isdir('junk.txt')

False

os.path.expanduser('~/local')

'/Users/mb312/local'

os.path.join(os.path.expanduser('~'), 'local', 'bin')

'/Users/mb312/local/bin'

Running an external command#

return_code = os.system('ls')

__pycache__
basic_types.md
control_flow.md
data.txt
demo.py
demo2.py
exceptions.md
first_steps.md
functions.md
io.md
junk.txt
my_file.py
oop.md
python_language.md
python-logo.png
reusing_code.md
solutions
standard_library.md
test.pkl
test.py
workfile

Note

Alternative to os.system

A noteworthy alternative to os.system is the sh module. Which provides much more convenient ways to obtain the output, error stream and exit code of the external command.

import sh
com = sh.ls()

print(com)
basic_types.md    exceptions.md   oop.md               standard_library.md
control_flow.md   first_steps.md  python_language.md
demo2.py          functions.md    python-logo.png
demo.py           io.md           reusing_code.md

type(com)
Out[33]: str

Walking a directory#

os.path.walk generates a list of filenames in a directory tree.

for dirpath, dirnames, filenames in os.walk(os.curdir):
    for fp in filenames:
        print(os.path.abspath(fp))

/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/oop.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/exceptions.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/basic_types.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/standard_library.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/junk.txt
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/python-logo.png
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/demo2.py
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/workfile
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/test.py
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/first_steps.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/test.pkl
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/functions.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/my_file.py
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/control_flow.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/demo.py
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/python_language.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/data.txt
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/io.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/reusing_code.md
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/test_dir_sort.py
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/dir_sort.py
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/path_site.py
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/demo2.cpython-312.pyc
/Volumes/zorg/mb312/dev_trees/scientific-python-lectures/intro/language/demo.cpython-312.pyc

Environment variables:#

In [2]: os.environ.keys()
Out[2]: KeysView(environ({'SHELL': '/bin/bash', 'PWD': '/home/mb312', 'LOGNAME': 'mb312', 'HOME': '/home/mb312', 'TERM': 'xterm', 'USER': 'mb312', 'SHLVL': '1', 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin', 'MAIL': '/var/mail/mb312', '_': '/usr/bin/python3', 'LC_CTYPE': 'C.UTF-8'}))

In [3]: os.environ['SHELL']
Out[3]: '/bin/bash'

`shutil`: high-level file operations#

The shutil provides useful file operations:

shutil.rmtree: Recursively delete a directory tree.
shutil.move: Recursively move a file or directory to another location.
shutil.copy: Copy files or directories.

`glob`: Pattern matching on files#

The glob module provides convenient file pattern matching.

Find all files ending in .txt:

import glob
glob.glob('*.txt')

['junk.txt', 'data.txt']

`sys` module: system-specific information#

System-specific information related to the Python interpreter.

Which version of Python are you running and where is it installed:

import sys
sys.platform

'darwin'

sys.version

'3.12.2 (v3.12.2:6abddd9f6a, Feb  6 2024, 17:02:06) [Clang 13.0.0 (clang-1300.0.29.30)]'

sys.prefix

'/Volumes/zorg/mb312/.virtualenvs/sp-lectures'

sys.argv gives you a list of command line arguments passed to a Python script. It is useful when you call as script with e.g. python my_script.py some arguments. Inside the my_arguments.py script, you can get the passed arguments (here [‘some’, ‘arguments’]) with sys.argv.

sys.path is a list of strings that specifies the search path for modules. Initialized from PYTHONPATH:

sys.path

['/Library/Frameworks/Python.framework/Versions/3.12/lib/python312.zip',
 '/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12',
 '/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/lib-dynload',
 '',
 '/Volumes/zorg/mb312/.virtualenvs/sp-lectures/lib/python3.12/site-packages',
 '/Volumes/zorg/mb312/dev_trees/jupytext/jupyterlab',
 '/Volumes/zorg/mb312/dev_trees/jupytext/src',
 '/Volumes/zorg/mb312/dev_trees/sphinx-book-theme/src']

`pickle`: easy persistence#

Useful to store arbitrary objects to a file. Not safe or fast!

import pickle
l = [1, None, 'Stan']
with open('test.pkl', 'wb') as file:
    pickle.dump(l, file)

with open('test.pkl', 'rb') as file:
    out = pickle.load(file)

out

[1, None, 'Stan']

Exercises#

Exercise 4

Write a function that will load the column of numbers in data.txt and calculate the min, max and sum values. Use no modules except those in the standard library; specifically, do not use Numpy.

data.txt:

Solution to Exercise 4

def load_data(filename):
    fp = open(filename)
    data_string = fp.read()
    fp.close()

    data = []
    for x in data_string.split():
        # Data is read in as a string. We need to convert it to floats
        data.append(float(x))

    # Could instead use the following one line with list comprehensions!
    # data = [float(x) for x in data_string.split()]
    return data

data = load_data("data.txt")
# Python provides these basic math functions.
print(f"min: {min(data):f}")
print(f"max: {max(data):f}")
print(f"sum: {sum(data):f}")

min: 10.200000
max: 61.300000
sum: 237.900000

Exercise 5

Implement a script that takes a directory name as argument, and returns the list of ‘.py’ files, sorted by name length.

Hint: try to understand the docstring of list.sort

Solution to Exercise 5

"""
Script to list all the '.py' files in a directory, in the order of file
name length.
"""

import os
import sys


def filter_and_sort(file_list):
    """Out of a list of file names, returns only the ones ending by
    '.py', ordered with increasing file name length.
    """
    file_list = [filename for filename in file_list if filename.endswith(".py")]

    def key(item):
        return len(item)

    file_list.sort(key=key)
    return file_list


if __name__ == "__main__":
    file_list = os.listdir(sys.argv[-1])
    sorted_file_list = filter_and_sort(file_list)
    print(sorted_file_list)

Exercise 6

Write a program to search your PYTHONPATH for the module site.py.

Solution to Exercise 6

"""Script to search the PYTHONPATH for the module site.py"""

import os
import sys
import glob


def find_module(module):
    result = []
    # Loop over the list of paths in sys.path
    for subdir in sys.path:
        # Join the subdir path with the module we're searching for
        pth = os.path.join(subdir, module)
        # Use glob to test if the pth is exists
        res = glob.glob(pth)
        # glob returns a list, if it is not empty, the pth exists
        if len(res) > 0:
            result.append(res)
    return result


if __name__ == "__main__":
    result = find_module("site.py")
    print(result)

Standard Library

Contents

Standard Library#

os module: operating system functionality#