Commit 4cce9c6c authored by katharina.berger's avatar katharina.berger

added option to change the permissions for retracted data on the filesystem

parent 61ce2dd6
......@@ -3,3 +3,4 @@ from mail_utils import send_mail
from solr_utils import query_solr, update_solr, query_esgf, get_list_of_shards, query_for_facet
from mapfile_utils import run_esgmapfile
from publish_utils import run_esgunpublish
from filesystem_utils import change_permissions_recursive
......@@ -4,11 +4,12 @@ import datetime
import getopt
import shutil
import time
import stat
import sys
import os
import re
from pkg_utils import query_solr, update_solr, query_esgf, send_mail, get_list_of_shards, run_esgmapfile, query_for_facet, run_esgunpublish
from pkg_utils import change_permissions_recursive
from config import Log, Config
from ESGConfigParser import SectionParser, interpolate
from pyesgf.search import SearchConnection
......@@ -26,6 +27,8 @@ usage = """
-c, --config config_file:
Configuration file
Use the default '/esg/config/replica_check.conf', if not specified.
--change-permissions:
Change the permissions of retracted datasets.
-p, --project project:
Check only data for the specified project, this option is mandatory.
--send-mail email_address[es]
......@@ -46,6 +49,8 @@ def get_deleted_and_retracted(log,
scan_directory,
retracted_list,
deleted_list,
to_unpublish_list,
change_permissions,
recipients=None):
"""
Get a list of all datasets that are on disk but either deleted or retracted in ESGF
......@@ -56,6 +61,13 @@ def get_deleted_and_retracted(log,
# get a list of all datasets on filesystem
for root, _, files in os.walk(scan_directory):
if files:
# check permissions, check only if not already set to "770"
st = os.stat(root)
permissions_other = oct(st.st_mode)[-1]
if permissions_other < 4:
continue
kv = {}
f = files[0]
if files:
......@@ -78,16 +90,21 @@ def get_deleted_and_retracted(log,
retracted = ctx.facet_counts['retracted'].keys()[0]
if retracted:
if retracted == 'true':
retracted_list.write('%s\n' % instance_id)
retracted_list.write('%s\n' % root)
if change_permissions:
change_permissions_recursive(root)
to_unpublish_list.write('%s#%s\n' % (master_id, kv['version'][1:]))
else:
deleted_list.write('%s\n' % instance_id)
deleted_list.write('%s\n' % root)
except:
log.info("\tCheck failed for=%s" % ffp)
def main(argv):
try:
args, lastargs = getopt.getopt(argv, "p:l:c:o:", ['config=', 'project=', 'send-mail=', 'log=', 'output='])
args, lastargs = getopt.getopt(argv, "p:l:c:o:", ['config=', 'change-permissions', 'project=', 'send-mail=', 'log=', 'output='])
except getopt.error:
print sys.exc_value
print usage
......@@ -99,6 +116,7 @@ def main(argv):
config_dir = '/esg/config/esgcet/'
config_filename = 'replica_check.conf'
output_folder = '/tmp'
change_permissions = False
for flag, arg in args:
if flag in ['-p', '--project']:
......@@ -111,6 +129,8 @@ def main(argv):
logfile_path = arg
elif flag in ['-o', '--output']:
output_folder = arg
elif flag == '--change-permissions':
change_permissions = True
if len(lastargs) != 0 and os.path.isdir(lastargs[0]):
path_to_scan = lastargs[0]
......@@ -136,18 +156,29 @@ def main(argv):
log = Log(logfile_name, 'INFO')
print('Logging information will be written to %s' % logfile_name)
last_dir = os.path.basename(os.path.normpath(path_to_scan))
project_config = SectionParser(section='project:%s' % project.lower(), directory=config_dir)
deleted_list = open(os.path.join(output_folder, 'deleted_datasets.txt'), 'w')
retracted_list = open(os.path.join(output_folder, 'retracted_datasets.txt'), 'w')
output_folder_version = os.path.join(output_folder, datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d'))
if not os.path.exists(output_folder_version):
os.makedirs(output_folder_version)
deleted_list = open(os.path.join(output_folder_version, 'deleted_datasets_%s.txt' % last_dir), 'w')
retracted_list = open(os.path.join(output_folder_version, 'retracted_datasets_%s.txt' % last_dir), 'w')
to_unpublish_list = open(os.path.join(output_folder_version, 'datasets_to_unpublish_%s.txt' % last_dir), 'w')
get_deleted_and_retracted(log,
project,
project_config,
path_to_scan,
retracted_list,
deleted_list,
to_unpublish_list,
change_permissions,
recipients=recipients)
retracted_list.close()
deleted_list.close()
to_unpublish_list.close()
if __name__ == '__main__':
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment