312 lines
11 KiB
Python
Executable File
312 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Copyright (C) 2014 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
import sys
|
|
import os
|
|
import os.path
|
|
import argparse
|
|
import csv
|
|
import collections
|
|
import math
|
|
|
|
try:
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.font_manager as fm
|
|
import matplotlib.pyplot as plt
|
|
except ImportError:
|
|
sys.stderr.write("You need python-matplotlib to generate the size graph\n")
|
|
exit(1)
|
|
|
|
|
|
class Config:
|
|
biggest_first = False
|
|
iec = False
|
|
size_limit = 0.01
|
|
colors = ['#e60004', '#f28e00', '#ffed00', '#940084',
|
|
'#2e1d86', '#0068b5', '#009836', '#97c000']
|
|
|
|
|
|
#
|
|
# This function adds a new file to 'filesdict', after checking its
|
|
# size. The 'filesdict' contain the relative path of the file as the
|
|
# key, and as the value a tuple containing the name of the package to
|
|
# which the file belongs and the size of the file.
|
|
#
|
|
# filesdict: the dict to which the file is added
|
|
# relpath: relative path of the file
|
|
# fullpath: absolute path to the file
|
|
# pkg: package to which the file belongs
|
|
#
|
|
def add_file(filesdict, relpath, abspath, pkg):
|
|
if relpath.endswith(".py"):
|
|
# also check for compiled .pyc file
|
|
add_file(filesdict, relpath + "c", abspath + "c", pkg)
|
|
if not os.path.exists(abspath):
|
|
return
|
|
if os.path.islink(abspath):
|
|
return
|
|
sz = os.stat(abspath).st_size
|
|
filesdict[relpath] = (pkg, sz)
|
|
|
|
|
|
#
|
|
# This function returns a dict where each key is the path of a file in
|
|
# the root filesystem, and the value is a tuple containing two
|
|
# elements: the name of the package to which this file belongs and the
|
|
# size of the file.
|
|
#
|
|
# builddir: path to the Buildroot output directory
|
|
#
|
|
def build_package_dict(builddir):
|
|
filesdict = {}
|
|
with open(os.path.join(builddir, "build", "packages-file-list.txt")) as f:
|
|
for line in f.readlines():
|
|
pkg, fpath = line.split(",", 1)
|
|
# remove the initial './' in each file path
|
|
fpath = fpath.strip()[2:]
|
|
fullpath = os.path.join(builddir, "target", fpath)
|
|
add_file(filesdict, fpath, fullpath, pkg)
|
|
return filesdict
|
|
|
|
|
|
#
|
|
# This function builds a dictionary that contains the name of a
|
|
# package as key, and the size of the files installed by this package
|
|
# as the value.
|
|
#
|
|
# filesdict: dictionary with the name of the files as key, and as
|
|
# value a tuple containing the name of the package to which the files
|
|
# belongs, and the size of the file. As returned by
|
|
# build_package_dict.
|
|
#
|
|
# builddir: path to the Buildroot output directory
|
|
#
|
|
def build_package_size(filesdict, builddir):
|
|
pkgsize = collections.defaultdict(int)
|
|
|
|
seeninodes = set()
|
|
for root, _, files in os.walk(os.path.join(builddir, "target")):
|
|
for f in files:
|
|
fpath = os.path.join(root, f)
|
|
if os.path.islink(fpath):
|
|
continue
|
|
|
|
st = os.stat(fpath)
|
|
if st.st_ino in seeninodes:
|
|
# hard link
|
|
continue
|
|
else:
|
|
seeninodes.add(st.st_ino)
|
|
|
|
frelpath = os.path.relpath(fpath, os.path.join(builddir, "target"))
|
|
if frelpath not in filesdict:
|
|
print("WARNING: %s is not part of any package" % frelpath)
|
|
pkg = "unknown"
|
|
else:
|
|
pkg = filesdict[frelpath][0]
|
|
|
|
pkgsize[pkg] += st.st_size
|
|
|
|
return pkgsize
|
|
|
|
|
|
#
|
|
# Given a dict returned by build_package_size(), this function
|
|
# generates a pie chart of the size installed by each package.
|
|
#
|
|
# pkgsize: dictionary with the name of the package as a key, and the
|
|
# size as the value, as returned by build_package_size.
|
|
#
|
|
# outputf: output file for the graph
|
|
#
|
|
def draw_graph(pkgsize, outputf):
|
|
def size2string(sz):
|
|
if Config.iec:
|
|
divider = 1024.0
|
|
prefixes = ['', 'Ki', 'Mi', 'Gi', 'Ti']
|
|
else:
|
|
divider = 1000.0
|
|
prefixes = ['', 'k', 'M', 'G', 'T']
|
|
while sz > divider and len(prefixes) > 1:
|
|
prefixes = prefixes[1:]
|
|
sz = sz/divider
|
|
# precision is made so that there are always at least three meaningful
|
|
# digits displayed (e.g. '3.14' and '10.4', not just '3' and '10')
|
|
precision = int(2-math.floor(math.log10(sz))) if sz < 1000 else 0
|
|
return '{:.{prec}f} {}B'.format(sz, prefixes[0], prec=precision)
|
|
|
|
total = sum(pkgsize.values())
|
|
labels = []
|
|
values = []
|
|
other_value = 0
|
|
unknown_value = 0
|
|
for (p, sz) in sorted(pkgsize.items(), key=lambda x: x[1],
|
|
reverse=Config.biggest_first):
|
|
if sz < (total * Config.size_limit):
|
|
other_value += sz
|
|
elif p == "unknown":
|
|
unknown_value = sz
|
|
else:
|
|
labels.append("%s (%s)" % (p, size2string(sz)))
|
|
values.append(sz)
|
|
if unknown_value != 0:
|
|
labels.append("Unknown (%s)" % (size2string(unknown_value)))
|
|
values.append(unknown_value)
|
|
if other_value != 0:
|
|
labels.append("Other (%s)" % (size2string(other_value)))
|
|
values.append(other_value)
|
|
|
|
plt.figure()
|
|
patches, texts, autotexts = plt.pie(values, labels=labels,
|
|
autopct='%1.1f%%', shadow=True,
|
|
colors=Config.colors)
|
|
# Reduce text size
|
|
proptease = fm.FontProperties()
|
|
proptease.set_size('xx-small')
|
|
plt.setp(autotexts, fontproperties=proptease)
|
|
plt.setp(texts, fontproperties=proptease)
|
|
|
|
plt.suptitle("Filesystem size per package", fontsize=18, y=.97)
|
|
plt.title("Total filesystem size: %s" % (size2string(total)), fontsize=10,
|
|
y=.96)
|
|
plt.savefig(outputf)
|
|
|
|
|
|
#
|
|
# Generate a CSV file with statistics about the size of each file, its
|
|
# size contribution to the package and to the overall system.
|
|
#
|
|
# filesdict: dictionary with the name of the files as key, and as
|
|
# value a tuple containing the name of the package to which the files
|
|
# belongs, and the size of the file. As returned by
|
|
# build_package_dict.
|
|
#
|
|
# pkgsize: dictionary with the name of the package as a key, and the
|
|
# size as the value, as returned by build_package_size.
|
|
#
|
|
# outputf: output CSV file
|
|
#
|
|
def gen_files_csv(filesdict, pkgsizes, outputf):
|
|
total = 0
|
|
for (p, sz) in pkgsizes.items():
|
|
total += sz
|
|
with open(outputf, 'w') as csvfile:
|
|
wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
|
|
wr.writerow(["File name",
|
|
"Package name",
|
|
"File size",
|
|
"Package size",
|
|
"File size in package (%)",
|
|
"File size in system (%)"])
|
|
for f, (pkgname, filesize) in filesdict.items():
|
|
pkgsize = pkgsizes[pkgname]
|
|
|
|
if pkgsize == 0:
|
|
percent_pkg = 0
|
|
else:
|
|
percent_pkg = float(filesize) / pkgsize * 100
|
|
|
|
percent_total = float(filesize) / total * 100
|
|
|
|
wr.writerow([f, pkgname, filesize, pkgsize,
|
|
"%.1f" % percent_pkg,
|
|
"%.1f" % percent_total])
|
|
|
|
|
|
#
|
|
# Generate a CSV file with statistics about the size of each package,
|
|
# and their size contribution to the overall system.
|
|
#
|
|
# pkgsize: dictionary with the name of the package as a key, and the
|
|
# size as the value, as returned by build_package_size.
|
|
#
|
|
# outputf: output CSV file
|
|
#
|
|
def gen_packages_csv(pkgsizes, outputf):
|
|
total = sum(pkgsizes.values())
|
|
with open(outputf, 'w') as csvfile:
|
|
wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
|
|
wr.writerow(["Package name", "Package size",
|
|
"Package size in system (%)"])
|
|
for (pkg, size) in pkgsizes.items():
|
|
wr.writerow([pkg, size, "%.1f" % (float(size) / total * 100)])
|
|
|
|
|
|
#
|
|
# Our special action for --iec, --binary, --si, --decimal
|
|
#
|
|
class PrefixAction(argparse.Action):
|
|
def __init__(self, option_strings, dest, **kwargs):
|
|
for key in ["type", "nargs"]:
|
|
if key in kwargs:
|
|
raise ValueError('"{}" not allowed'.format(key))
|
|
super(PrefixAction, self).__init__(option_strings, dest, nargs=0,
|
|
type=bool, **kwargs)
|
|
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
setattr(namespace, self.dest, option_string in ["--iec", "--binary"])
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Draw size statistics graphs')
|
|
|
|
parser.add_argument("--builddir", '-i', metavar="BUILDDIR", required=True,
|
|
help="Buildroot output directory")
|
|
parser.add_argument("--graph", '-g', metavar="GRAPH",
|
|
help="Graph output file (.pdf or .png extension)")
|
|
parser.add_argument("--file-size-csv", '-f', metavar="FILE_SIZE_CSV",
|
|
help="CSV output file with file size statistics")
|
|
parser.add_argument("--package-size-csv", '-p', metavar="PKG_SIZE_CSV",
|
|
help="CSV output file with package size statistics")
|
|
parser.add_argument("--biggest-first", action='store_true',
|
|
help="Sort packages in decreasing size order, " +
|
|
"rather than in increasing size order")
|
|
parser.add_argument("--iec", "--binary", "--si", "--decimal",
|
|
action=PrefixAction,
|
|
help="Use IEC (binary, powers of 1024) or SI (decimal, "
|
|
"powers of 1000, the default) prefixes")
|
|
parser.add_argument("--size-limit", "-l", type=float,
|
|
help='Under this size ratio, files are accounted to ' +
|
|
'the generic "Other" package. Default: 0.01 (1%%)')
|
|
args = parser.parse_args()
|
|
|
|
Config.biggest_first = args.biggest_first
|
|
Config.iec = args.iec
|
|
if args.size_limit is not None:
|
|
if args.size_limit < 0.0 or args.size_limit > 1.0:
|
|
parser.error("--size-limit must be in [0.0..1.0]")
|
|
Config.size_limit = args.size_limit
|
|
|
|
# Find out which package installed what files
|
|
pkgdict = build_package_dict(args.builddir)
|
|
|
|
# Collect the size installed by each package
|
|
pkgsize = build_package_size(pkgdict, args.builddir)
|
|
|
|
if args.graph:
|
|
draw_graph(pkgsize, args.graph)
|
|
if args.file_size_csv:
|
|
gen_files_csv(pkgdict, pkgsize, args.file_size_csv)
|
|
if args.package_size_csv:
|
|
gen_packages_csv(pkgsize, args.package_size_csv)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|