mirror of
https://github.com/gluster/glusterfs.git
synced 2026-02-05 15:48:40 +01:00
89a34b574e
Following files are fixed. tests/bugs/distribute/overlap.py tests/utils/changelogparser.py tests/utils/create-files.py tests/utils/gfid-access.py tests/utils/libcxattr.py Change-Id: I3db857cc19e19163d368d913eaec1269fbc37140 updates: bz#1193929 Signed-off-by: Kotresh HR <khiremat@redhat.com>
594 lines
22 KiB
Python
Executable File
594 lines
22 KiB
Python
Executable File
|
|
# This script was developed by Vijaykumar Koppad (vkoppad@redhat.com)
|
|
# The latest version of this script can found at
|
|
# http://github.com/vijaykumar-koppad/crefi
|
|
|
|
from __future__ import with_statement
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import errno
|
|
import xattr
|
|
import string
|
|
import random
|
|
import logging
|
|
import tarfile
|
|
import argparse
|
|
|
|
datsiz = 0
|
|
timr = 0
|
|
|
|
def get_ascii_upper_alpha_digits():
|
|
if sys.version_info > (3,0):
|
|
return string.ascii_uppercase+string.digits
|
|
else:
|
|
return string.uppercase+string.digits
|
|
|
|
def setLogger(filename):
|
|
global logger
|
|
logger = logging.getLogger(filename)
|
|
logger.setLevel(logging.DEBUG)
|
|
return
|
|
|
|
|
|
def setupLogger(filename):
|
|
logger = logging.getLogger(filename)
|
|
logger.setLevel(logging.DEBUG)
|
|
formatter = logging.Formatter('%(asctime)s - %(message)s')
|
|
ch = logging.StreamHandler()
|
|
ch.setLevel(logging.INFO)
|
|
ch.setFormatter(formatter)
|
|
logger.addHandler(ch)
|
|
return logger
|
|
|
|
|
|
def os_rd(src, size):
|
|
global datsiz
|
|
fd = os.open(src, os.O_RDONLY)
|
|
data = os.read(fd, size)
|
|
os.close(fd)
|
|
datsiz = datsiz + size
|
|
return data
|
|
|
|
|
|
def os_wr(dest, data):
|
|
global timr
|
|
st = time.time()
|
|
fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
os.write(fd, data)
|
|
os.close(fd)
|
|
ed = time.time()
|
|
timr = timr+(ed-st)
|
|
return
|
|
|
|
|
|
def create_sparse_file(fil, size, mins, maxs, rand):
|
|
if rand:
|
|
size = random.randint(mins, maxs)
|
|
else:
|
|
size = size
|
|
data = os_rd("/dev/zero", size)
|
|
os_wr(fil, data)
|
|
return
|
|
|
|
|
|
def create_binary_file(fil, size, mins, maxs, rand):
|
|
if rand:
|
|
size = random.randint(mins, maxs)
|
|
else:
|
|
size = size
|
|
data = os_rd("/dev/urandom", size)
|
|
os_wr(fil, data)
|
|
return
|
|
|
|
|
|
def create_txt_file(fil, size, mins, maxs, rand):
|
|
if rand:
|
|
size = random.randint(mins, maxs)
|
|
if size < 500*1024:
|
|
data = os_rd("/etc/services", size)
|
|
os_wr(fil, data)
|
|
else:
|
|
data = os_rd("/etc/services", 512*1024)
|
|
file_size = 0
|
|
fd = os.open(fil, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
while file_size < size:
|
|
os.write(fd, data)
|
|
file_size += 500*1024
|
|
os.close(fd)
|
|
return
|
|
|
|
|
|
def create_tar_file(fil, size, mins, maxs, rand):
|
|
if rand:
|
|
size = random.randint(mins, maxs)
|
|
else:
|
|
size = size
|
|
data = os_rd("/dev/urandom", size)
|
|
os_wr(fil, data)
|
|
tar = tarfile.open(fil+".tar.gz", "w:gz")
|
|
tar.add(fil)
|
|
tar.close()
|
|
os.unlink(fil)
|
|
return
|
|
|
|
|
|
def get_filename(flen):
|
|
size = flen
|
|
char = get_ascii_upper_alpha_digits()
|
|
st = ''.join(random.choice(char) for i in range(size))
|
|
ti = str((hex(int(str(time.time()).split('.')[0])))[2:])
|
|
return ti+"%%"+st
|
|
|
|
|
|
def text_files(files, file_count, inter, size, mins, maxs, rand,
|
|
flen, randname, dir_path):
|
|
global datsiz, timr
|
|
for k in range(files):
|
|
if not file_count % inter:
|
|
logger.info("Total files created -- "+str(file_count))
|
|
if not randname:
|
|
fil = dir_path+"/"+"file"+str(k)
|
|
else:
|
|
fil = dir_path+"/"+get_filename(flen)
|
|
create_txt_file(fil, size, mins, maxs, rand)
|
|
file_count += 1
|
|
return file_count
|
|
|
|
|
|
def sparse_files(files, file_count, inter, size, mins, maxs,
|
|
rand, flen, randname, dir_path):
|
|
for k in range(files):
|
|
if not file_count % inter:
|
|
logger.info("Total files created -- "+str(file_count))
|
|
if not randname:
|
|
fil = dir_path+"/"+"file"+str(k)
|
|
else:
|
|
fil = dir_path+"/"+get_filename(flen)
|
|
create_sparse_file(fil, size, mins, maxs, rand)
|
|
file_count += 1
|
|
return file_count
|
|
|
|
|
|
def binary_files(files, file_count, inter, size, mins, maxs,
|
|
rand, flen, randname, dir_path):
|
|
for k in range(files):
|
|
if not file_count % inter:
|
|
logger.info("Total files created -- "+str(file_count))
|
|
if not randname:
|
|
fil = dir_path+"/"+"file"+str(k)
|
|
else:
|
|
fil = dir_path+"/"+get_filename(flen)
|
|
create_binary_file(fil, size, mins, maxs, rand)
|
|
file_count += 1
|
|
return file_count
|
|
|
|
|
|
def tar_files(files, file_count, inter, size, mins, maxs,
|
|
rand, flen, randname, dir_path):
|
|
for k in range(files):
|
|
if not file_count % inter:
|
|
logger.info("Total files created -- "+str(file_count))
|
|
if not randname:
|
|
fil = dir_path+"/"+"file"+str(k)
|
|
else:
|
|
fil = dir_path+"/"+get_filename(flen)
|
|
create_tar_file(fil, size, mins, maxs, rand)
|
|
file_count += 1
|
|
return file_count
|
|
|
|
|
|
def setxattr_files(files, randname, dir_path):
|
|
char = get_ascii_upper_alpha_digits()
|
|
if not randname:
|
|
for k in range(files):
|
|
v = ''.join(random.choice(char) for i in range(10))
|
|
n = "user."+v
|
|
xattr.setxattr(dir_path+"/"+"file"+str(k), n, v)
|
|
else:
|
|
dirs = os.listdir(dir_path+"/")
|
|
for fil in dirs:
|
|
v = ''.join(random.choice(char) for i in range(10))
|
|
n = "user."+v
|
|
xattr.setxattr(dir_path+"/"+fil, n, v)
|
|
return
|
|
|
|
|
|
def rename_files(files, flen, randname, dir_path):
|
|
if not randname:
|
|
for k in range(files):
|
|
os.rename(dir_path + "/" + "file" + str(k),
|
|
dir_path + "/" + "file" + str(files+k))
|
|
else:
|
|
dirs = os.listdir(dir_path)
|
|
for fil in dirs:
|
|
if not os.path.isdir(fil):
|
|
newfil = get_filename(flen)
|
|
os.rename(dir_path + "/" + fil,
|
|
dir_path + "/" + newfil)
|
|
return
|
|
|
|
|
|
def truncate_files(files, mins, maxs, randname, dir_path):
|
|
if not randname:
|
|
for k in range(files):
|
|
byts = random.randint(mins, maxs)
|
|
fd = os.open(dir_path + "/" + "file" + str(k), os.O_WRONLY)
|
|
os.ftruncate(fd, byts)
|
|
os.close(fd)
|
|
else:
|
|
dirs = os.listdir(dir_path)
|
|
for fil in dirs:
|
|
if not os.path.isdir(dir_path+"/"+fil):
|
|
byts = random.randint(mins, maxs)
|
|
fd = os.open(dir_path+"/"+fil, os.O_WRONLY)
|
|
os.ftruncate(fd, byts)
|
|
os.close(fd)
|
|
return
|
|
|
|
|
|
def chmod_files(files, flen, randname, dir_path):
|
|
if not randname:
|
|
for k in range(files):
|
|
mod = random.randint(0, 511)
|
|
os.chmod(dir_path+"/"+"file"+str(k), mod)
|
|
else:
|
|
dirs = os.listdir(dir_path)
|
|
for fil in dirs:
|
|
mod = random.randint(0, 511)
|
|
os.chmod(dir_path+"/"+fil, mod)
|
|
return
|
|
|
|
def random_og(path):
|
|
u = random.randint(1025, 65536)
|
|
g = -1
|
|
os.chown(path, u, g)
|
|
|
|
def chown_files(files, flen, randname, dir_path):
|
|
if not randname:
|
|
for k in range(files):
|
|
random_og(dir_path+"/"+"file"+str(k))
|
|
else:
|
|
dirs = os.listdir(dir_path)
|
|
for fil in dirs:
|
|
random_og(dir_path+"/"+fil)
|
|
return
|
|
|
|
|
|
def chgrp_files(files, flen, randname, dir_path):
|
|
if not randname:
|
|
for k in range(files):
|
|
random_og(dir_path+"/"+"file"+str(k))
|
|
else:
|
|
dirs = os.listdir(dir_path)
|
|
for fil in dirs:
|
|
random_og(dir_path+"/"+fil)
|
|
return
|
|
|
|
|
|
def symlink_files(files, flen, randname, dir_path):
|
|
try:
|
|
os.makedirs(dir_path+"/"+"symlink_to_files")
|
|
except OSError as ex:
|
|
if ex.errno is not errno.EEXIST:
|
|
raise
|
|
if not randname:
|
|
for k in range(files):
|
|
src_file = "file"+str(k)
|
|
os.symlink(dir_path+"/"+src_file,
|
|
dir_path+"/"+"symlink_to_files/file"+str(k)+"_sym")
|
|
else:
|
|
dirs = os.listdir(dir_path)
|
|
for fil in dirs:
|
|
newfil = get_filename(flen)
|
|
os.symlink(dir_path+"/"+fil,
|
|
dir_path+"/"+"symlink_to_files/"+newfil)
|
|
return
|
|
|
|
|
|
def hardlink_files(files, flen, randname, dir_path):
|
|
try:
|
|
os.makedirs(dir_path+"/"+"hardlink_to_files")
|
|
except OSError as ex:
|
|
if ex.errno is not errno.EEXIST:
|
|
raise
|
|
if not randname:
|
|
for k in range(files):
|
|
src_file = "file"+str(k)
|
|
os.link(dir_path+"/"+src_file,
|
|
dir_path+"/"+"hardlink_to_files/file"+str(k)+"_hard")
|
|
else:
|
|
dirs = os.listdir(dir_path)
|
|
for fil in dirs:
|
|
if not os.path.isdir(dir_path+"/"+fil):
|
|
newfil = get_filename(flen)
|
|
os.link(dir_path+"/"+fil,
|
|
dir_path+"/"+"hardlink_to_files/"+newfil)
|
|
return
|
|
|
|
|
|
def human2bytes(size):
|
|
size_short = {
|
|
1024: ['K', 'KB', 'KiB', 'k', 'kB', 'kiB'],
|
|
1024*1024: ['M', 'MB', 'MiB'],
|
|
1024*1024*1024: ['G', 'GB', 'GiB']
|
|
}
|
|
num = re.search('(\d+)', size).group()
|
|
ext = size[len(num):]
|
|
num = int(num)
|
|
if ext == '':
|
|
return num
|
|
for value, keys in size_short.items():
|
|
if ext in keys:
|
|
size = num*value
|
|
return size
|
|
|
|
|
|
def bytes2human(byts):
|
|
abbr = {
|
|
1 << 30: "GB",
|
|
1 << 20: "MB",
|
|
1 << 10: "KB",
|
|
1: "bytes"
|
|
}
|
|
if byts == 1:
|
|
return '1 bytes'
|
|
for factor, suffix in abbr.items():
|
|
if byts >= factor:
|
|
break
|
|
return "%.3f %s" % (byts / factor, suffix)
|
|
|
|
|
|
def multipledir(mnt_pnt, brdth, depth, files, fop, file_type="text",
|
|
inter="1000", size="100K", mins="10K", maxs="500K",
|
|
rand=False, l=10, randname=False):
|
|
files_count = 1
|
|
size = human2bytes(size)
|
|
maxs = human2bytes(maxs)
|
|
mins = human2bytes(mins)
|
|
for i in range(brdth):
|
|
dir_path = mnt_pnt
|
|
for j in range(depth):
|
|
dir_path = dir_path+"/"+"level"+str(j)+str(i)
|
|
try:
|
|
os.makedirs(dir_path)
|
|
except OSError as ex:
|
|
if ex.errno is not errno.EEXIST:
|
|
raise
|
|
|
|
if fop == "create":
|
|
logger.info("Entering the directory level"+str(j)+str(i))
|
|
if file_type == "text":
|
|
files_count = text_files(files, files_count, inter, size,
|
|
mins, maxs, rand, l, randname,
|
|
dir_path)
|
|
elif file_type == "sparse":
|
|
files_count = sparse_files(files, files_count, inter, size,
|
|
mins, maxs, rand, l, randname,
|
|
dir_path)
|
|
elif file_type == "binary":
|
|
files_count = binary_files(files, files_count, inter, size,
|
|
mins, maxs, rand, l, randname,
|
|
dir_path)
|
|
elif file_type == "tar":
|
|
files_count = tar_files(files, files_count, inter, size,
|
|
mins, maxs, rand, l, randname,
|
|
dir_path)
|
|
else:
|
|
logger.error("Not a valid file type")
|
|
sys.exit(1)
|
|
|
|
elif fop == "rename":
|
|
logger.info("Started renaming files for the files 0 to " +
|
|
str(files)+" in the directory level"+str(j) +
|
|
str(i)+" ...")
|
|
rename_files(files, l, randname, dir_path)
|
|
logger.info("Finished renaming files for the files 0 to " +
|
|
str(files)+" in the directory level"+str(j)+str(i))
|
|
|
|
elif fop == "chmod":
|
|
logger.info("Started changing permission of files for the " +
|
|
"files 0 to "+str(files)+" in the directory level"
|
|
+ str(j)+str(i)+" ...")
|
|
chmod_files(files, l, randname, dir_path)
|
|
logger.info("Finished changing permission of files for " +
|
|
"the files 0 to "+str(files) +
|
|
" in the directory level"+str(j)+str(i))
|
|
|
|
elif fop == "chown":
|
|
logger.info("Started changing ownership of files for the " +
|
|
"files 0 to " + str(files) +
|
|
" in the directory level"+str(j)+str(i)+" ...")
|
|
chown_files(files, l, randname, dir_path)
|
|
logger.info("Finished changing ownership of files for " +
|
|
"the files 0 to "+str(files) +
|
|
" in the directory level"+str(j)+str(i))
|
|
|
|
elif fop == "chgrp":
|
|
logger.info("Started changing group ownership of files for " +
|
|
"the files 0 to " + str(files) +
|
|
" in the directory level"+str(j)+str(i)+" ...")
|
|
chgrp_files(files, l, randname, dir_path)
|
|
logger.info("Finished changing group ownership of files for " +
|
|
"the files 0 to "+str(files) +
|
|
" in the directory level"+str(j)+str(i))
|
|
|
|
elif fop == "symlink":
|
|
logger.info("Started creating symlink to the files 0 to " +
|
|
str(files)+" in the directory level" +
|
|
str(j)+str(i)+"...")
|
|
symlink_files(files, l, randname, dir_path)
|
|
logger.info("Finished creating symlink to the files 0 to " +
|
|
str(files) + " in the directory level" +
|
|
str(j)+str(i))
|
|
|
|
elif fop == "hardlink":
|
|
logger.info("Started creating hardlink to the files 0 to " +
|
|
str(files)+" in the directory level" +
|
|
str(j)+str(i)+"...")
|
|
hardlink_files(files, l, randname, dir_path)
|
|
logger.info("Finished creating hardlink to the files 0 to " +
|
|
str(files) + " in the directory level" +
|
|
str(j)+str(i))
|
|
|
|
elif fop == "truncate":
|
|
logger.info("Started truncating the files 0 to " +
|
|
str(files)+" in the directory level" +
|
|
str(j)+str(i)+"...")
|
|
truncate_files(files, mins, maxs, randname, dir_path)
|
|
logger.info("Finished truncating the files 0 to " +
|
|
str(files)+" in the directory level" +
|
|
str(j)+str(i))
|
|
|
|
elif fop == "setxattr":
|
|
logger.info("Started setxattr to the files 0 to " +
|
|
str(files)+" in the directory level" +
|
|
str(j)+str(i)+"...")
|
|
setxattr_files(files, randname, dir_path)
|
|
logger.info("Finished setxattr to the files 0 to " +
|
|
str(files)+" in the directory level" +
|
|
str(j)+str(i))
|
|
|
|
if fop == "create":
|
|
thrpt = datsiz / timr
|
|
logger.info("finished creating files with throughput ---- " +
|
|
bytes2human(thrpt)+"ps")
|
|
|
|
|
|
def singledir(mnt_pnt, files, fop, file_type="text", inter="1000", size="100K",
|
|
mins="10K", maxs="500K", rand=False, l=10, randname=False):
|
|
|
|
files_count = 1
|
|
size = human2bytes(size)
|
|
maxs = human2bytes(maxs)
|
|
mins = human2bytes(mins)
|
|
if fop == "create":
|
|
if file_type == "text":
|
|
files_count = text_files(files, files_count, inter, size, mins,
|
|
maxs, rand, l, randname, mnt_pnt)
|
|
elif file_type == "sparse":
|
|
files_count = sparse_files(files, files_count, inter, size, mins,
|
|
maxs, rand, l, randname, mnt_pnt)
|
|
elif file_type == "binary":
|
|
files_count = binary_files(files, files_count, inter, size, mins,
|
|
maxs, rand, l, randname, mnt_pnt)
|
|
elif file_type == "tar":
|
|
files_count = tar_files(files, files_count, inter, size, mins,
|
|
maxs, rand, l, randname, mnt_pnt)
|
|
else:
|
|
logger.info("Not a valid file type")
|
|
sys.exit(1)
|
|
thrpt = datsiz / timr
|
|
logger.info("finished creating files with avg throughput ---- " +
|
|
bytes2human(thrpt)+"ps")
|
|
|
|
elif fop == "rename":
|
|
logger.info("Started renaming files for the files 0 to " +
|
|
str(files) + "...")
|
|
rename_files(files, l, randname, mnt_pnt)
|
|
logger.info("Finished renaming files for the files 0 to "+str(files))
|
|
|
|
elif fop == "chmod":
|
|
logger.info("Started changing permission for the files 0 to " +
|
|
str(files)+" ...")
|
|
chmod_files(files, l, randname, mnt_pnt)
|
|
logger.info("Finished changing permission files for the files 0 to " +
|
|
str(files))
|
|
|
|
elif fop == "chown":
|
|
logger.info("Started changing ownership for the files 0 to " +
|
|
str(files)+"...")
|
|
chown_files(files, l, randname, mnt_pnt)
|
|
logger.info("Finished changing ownership for the files 0 to " +
|
|
str(files))
|
|
|
|
elif fop == "chgrp":
|
|
logger.info("Started changing group ownership for the files 0 to " +
|
|
str(files)+"...")
|
|
chgrp_files(files, l, randname, mnt_pnt)
|
|
logger.info("Finished changing group ownership for the files 0 to " +
|
|
str(files))
|
|
|
|
elif fop == "symlink":
|
|
logger.info("Started creating symlink to the files 0 to " +
|
|
str(files)+"...")
|
|
symlink_files(files, l, randname, mnt_pnt)
|
|
logger.info("Finished creating symlink to the files 0 to " +
|
|
str(files))
|
|
|
|
elif fop == "hardlink":
|
|
logger.info("Started creating hardlink to the files 0 to " +
|
|
str(files)+"...")
|
|
hardlink_files(files, l, randname, mnt_pnt)
|
|
logger.info("Finished creating hardlink to the files 0 to " +
|
|
str(files))
|
|
|
|
elif fop == "truncate":
|
|
logger.info("Started truncating the files 0 to " + str(files)+"...")
|
|
truncate_files(files, mins, maxs, randname, mnt_pnt)
|
|
logger.info("Finished truncating the files 0 to " + str(files))
|
|
|
|
elif fop == "setxattr":
|
|
logger.info("Started setxattr to the files 0 to " + str(files)+"...")
|
|
setxattr_files(files, randname, mnt_pnt)
|
|
logger.info("Finished setxattr to the files 0 to " + str(files))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
usage = "usage: %prog [option] <MNT_PT>"
|
|
parser = argparse.ArgumentParser(formatter_class=argparse.
|
|
ArgumentDefaultsHelpFormatter)
|
|
parser.add_argument("-n", dest="files", type=int, default=100,
|
|
help="number of files in each level ")
|
|
parser.add_argument("--size", action="store", default="100k",
|
|
help="size of the files to be used ")
|
|
parser.add_argument("--random", action="store_true", default=False,
|
|
help="random size of the file between --min and --max")
|
|
parser.add_argument("--max", action="store", default="500K",
|
|
help="maximum size of the files, if random is True")
|
|
parser.add_argument("--min", action="store", default="10K",
|
|
help="minimum size of the files, if random is True")
|
|
parser.add_argument("--single", action="store_true", dest="dir",
|
|
default=True, help="create files in single directory")
|
|
parser.add_argument("--multi", action="store_false", dest="dir",
|
|
help="create files in multiple directories")
|
|
parser.add_argument("-b", dest="brdth", type=int, default=5,
|
|
help="number of directories in one level(works " +
|
|
"with --multi) ")
|
|
parser.add_argument("-d", dest="depth", type=int, default=5,
|
|
help="number of levels of directories (works " +
|
|
"with --multi) ")
|
|
parser.add_argument("-l", dest="flen", type=int, default=10,
|
|
help="number of bytes for filename ( Used only when " +
|
|
"randname is enabled) ")
|
|
parser.add_argument("-t", action="store", dest="file_type",
|
|
default="text", choices=["text", "sparse", "binary",
|
|
"tar"],
|
|
help="type of the file to be created ()")
|
|
parser.add_argument("-I", dest="inter", type=int, default=100,
|
|
help="print number files created of interval")
|
|
parser.add_argument("--fop", action="store", dest="fop", default="create",
|
|
choices=["create", "rename", "chmod", "chown", "chgrp",
|
|
"symlink", "hardlink", "truncate",
|
|
"setxattr"],
|
|
help="fop to be performed on the files")
|
|
parser.add_argument("-R", dest="randname", action="store_false",
|
|
default=True, help="To disable random file name " +
|
|
"(default: Enabled)")
|
|
parser.add_argument("mntpnt", help="Mount point")
|
|
|
|
args = parser.parse_args()
|
|
logger = setupLogger("testlost")
|
|
args.mntpnt = os.path.abspath(args.mntpnt)
|
|
|
|
if args.dir:
|
|
singledir(args.mntpnt, args.files, args.fop, args.file_type,
|
|
args.inter, args.size, args.min, args.max,
|
|
args.random, args.flen, args.randname)
|
|
else:
|
|
multipledir(args.mntpnt, args.brdth, args.depth, args.files,
|
|
args.fop, args.file_type, args.inter, args.size,
|
|
args.min, args.max, args.random, args.flen,
|
|
args.randname)
|