# -*- coding: utf-8 -*- import os import re import uuid from .._compat import ( PY2, BytesIO, iteritems, integer_types, string_types, to_bytes, pjoin, exists ) from .regex import REGEX_NOPASSWD, REGEX_UNPACK, REGEX_CONST_STRING, REGEX_W from .classes import SQLCustomType UNIT_SEPARATOR = '\x1f' # ASCII unit separater for delimiting data PLURALIZE_RULES = [ (re.compile('child$'), re.compile('child$'), 'children'), (re.compile('oot$'), re.compile('oot$'), 'eet'), (re.compile('ooth$'), re.compile('ooth$'), 'eeth'), (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'), (re.compile('sis$'), re.compile('sis$'), 'ses'), (re.compile('man$'), re.compile('man$'), 'men'), (re.compile('ife$'), re.compile('ife$'), 'ives'), (re.compile('eau$'), re.compile('eau$'), 'eaux'), (re.compile('lf$'), re.compile('lf$'), 'lves'), (re.compile('[sxz]$'), re.compile('$'), 'es'), (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'), (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'), (re.compile('$'), re.compile('$'), 's'), ] def pluralize(singular, rules=PLURALIZE_RULES): for line in rules: re_search, re_sub, replace = line plural = re_search.search(singular) and re_sub.sub(replace, singular) if plural: return plural def hide_password(uri): if isinstance(uri, (list, tuple)): return [hide_password(item) for item in uri] return REGEX_NOPASSWD.sub('******', uri) def cleanup(text): """ Validates that the given text is clean: only contains [0-9a-zA-Z_] """ # if not REGEX_ALPHANUMERIC.match(text): # raise SyntaxError('invalid table or field name: %s' % text) return text def list_represent(values, row=None): return ', '.join(str(v) for v in (values or [])) def xorify(orderby): if not orderby: return None orderby2 = orderby[0] for item in orderby[1:]: orderby2 = orderby2 | item return orderby2 def use_common_filters(query): return (query and hasattr(query, 'ignore_common_filters') and \ not query.ignore_common_filters) def merge_tablemaps(*maplist): """ Merge arguments into a single dict, check for name collisions. """ maplist = list(maplist) for i, item in enumerate(maplist): if isinstance(item, dict): maplist[i] = dict(**item) ret = maplist[0] for item in maplist[1:]: if len(ret) > len(item): big, small = ret, item else: big, small = item, ret # Check for name collisions for key, val in small.items(): if big.get(key, val) is not val: raise ValueError('Name conflict in table list: %s' % key) # Merge big.update(small) ret = big return ret def bar_escape(item): item = str(item).replace('|', '||') if item.startswith('||'): item='%s%s' % (UNIT_SEPARATOR, item) if item.endswith('||'): item='%s%s' % (item, UNIT_SEPARATOR) return item def bar_unescape(item): item = item.replace('||','|') if item.startswith(UNIT_SEPARATOR): item = item[1:] if item.endswith(UNIT_SEPARATOR): item = item[:-1] return item def bar_encode(items): return '|%s|' % '|'.join(bar_escape(item) for item in items if str(item).strip()) def bar_decode_integer(value): long = integer_types[-1] if not hasattr(value, 'split') and hasattr(value, 'read'): value = value.read() return [long(x) for x in value.split('|') if x.strip()] def bar_decode_string(value): return [bar_unescape(x) for x in REGEX_UNPACK.split(value[1:-1]) if x.strip()] def archive_record(qset, fs, archive_table, current_record): tablenames = qset.db._adapter.tables(qset.query) if len(tablenames) != 1: raise RuntimeError("cannot update join") for row in qset.select(): fields = archive_table._filter_fields(row) for k, v in iteritems(fs): if fields[k] != v: fields[current_record] = row.id archive_table.insert(**fields) break return False def smart_query(fields, text): from ..objects import Field, Table if not isinstance(fields, (list, tuple)): fields = [fields] new_fields = [] for field in fields: if isinstance(field, Field): new_fields.append(field) elif isinstance(field, Table): for ofield in field: new_fields.append(ofield) else: raise RuntimeError("fields must be a list of fields") fields = new_fields field_map = {} for field in fields: n = field.name.lower() if not n in field_map: field_map[n] = field n = str(field).lower() if not n in field_map: field_map[n] = field constants = {} i = 0 while True: m = REGEX_CONST_STRING.search(text) if not m: break text = text[:m.start()]+('#%i' % i)+text[m.end():] constants[str(i)] = m.group()[1:-1] i += 1 text = re.sub('\s+', ' ', text).lower() for a, b in [('&', 'and'), ('|', 'or'), ('~', 'not'), ('==', '='), ('<', '<'), ('>', '>'), ('<=', '<='), ('>=', '>='), ('<>', '!='), ('=<', '<='), ('=>', '>='), ('=', '='), (' less or equal than ', '<='), (' greater or equal than ', '>='), (' equal or less than ', '<='), (' equal or greater than ', '>='), (' less or equal ', '<='), (' greater or equal ', '>='), (' equal or less ', '<='), (' equal or greater ', '>='), (' not equal to ', '!='), (' not equal ', '!='), (' equal to ', '='), (' equal ', '='), (' equals ', '='), (' less than ', '<'), (' greater than ', '>'), (' starts with ', 'startswith'), (' ends with ', 'endswith'), (' not in ', 'notbelongs'), (' in ', 'belongs'), (' is ', '=')]: if a[0] == ' ': text = text.replace(' is'+a, ' %s ' % b) text = text.replace(a, ' %s ' % b) text = re.sub('\s+', ' ', text).lower() text = re.sub('(?P[\<\>\!\=])\s+(?P[\<\>\!\=])', '\g\g', text) query = field = neg = op = logic = None for item in text.split(): if field is None: if item == 'not': neg = True elif not neg and not logic and item in ('and', 'or'): logic = item elif item in field_map: field = field_map[item] else: raise RuntimeError("Invalid syntax") elif not field is None and op is None: op = item elif not op is None: if item.startswith('#'): if not item[1:] in constants: raise RuntimeError("Invalid syntax") value = constants[item[1:]] else: value = item if field.type in ('text', 'string', 'json'): if op == '=': op = 'like' if op == '=': new_query = field == value elif op == '<': new_query = field < value elif op == '>': new_query = field > value elif op == '<=': new_query = field <= value elif op == '>=': new_query = field >= value elif op == '!=': new_query = field != value elif op == 'belongs': new_query = field.belongs(value.split(',')) elif op == 'notbelongs': new_query = ~field.belongs(value.split(',')) elif field.type == 'list:string': if op == 'contains': new_query = field.contains(value) else: raise RuntimeError("Invalid operation") elif field.type in ('text', 'string', 'json', 'upload'): if op == 'contains': new_query = field.contains(value) elif op == 'like': new_query = field.ilike(value) elif op == 'startswith': new_query = field.startswith(value) elif op == 'endswith': new_query = field.endswith(value) else: raise RuntimeError("Invalid operation") elif field._db._adapter.dbengine=='google:datastore' and \ field.type in ('list:integer', 'list:string', 'list:reference'): if op == 'contains': new_query = field.contains(value) else: raise RuntimeError("Invalid operation") else: raise RuntimeError("Invalid operation") if neg: new_query = ~new_query if query is None: query = new_query elif logic == 'and': query &= new_query elif logic == 'or': query |= new_query field = op = neg = logic = None return query def auto_validators(field): db = field.db field_type = field.type #: don't apply default validation on custom types if isinstance(field_type, SQLCustomType): if hasattr(field_type, 'validator'): return field_type.validator else: field_type = field_type.type elif not isinstance(field_type, str): return [] #: if a custom method is provided, call it if callable(db.validators_method): return db.validators_method(field) #: apply validators from validators dict if present if not db.validators or not isinstance(db.validators, dict): return [] field_validators = db.validators.get(field_type, []) if not isinstance(field_validators, (list, tuple)): field_validators = [field_validators] return field_validators def _fieldformat(r, id): row = r(id) if not row: return str(id) elif hasattr(r, '_format') and isinstance(r._format, str): return r._format % row elif hasattr(r, '_format') and callable(r._format): return r._format(row) else: return str(id) class _repr_ref(object): def __init__(self, ref=None): self.ref = ref def __call__(self, value, row=None): return value if value is None else _fieldformat(self.ref, value) class _repr_ref_list(_repr_ref): def __call__(self, value, row=None): if not value: return None refs = None db, id = self.ref._db, self.ref._id if db._adapter.dbengine == 'google:datastore': def count(values): return db(id.belongs(values)).select(id) rx = range(0, len(value), 30) refs = reduce(lambda a, b: a & b, [count(value[i:i+30]) for i in rx]) else: refs = db(id.belongs(value)).select(id) return refs and ', '.join( _fieldformat(self.ref, x) for x in value) or '' def auto_represent(field): if field.represent: return field.represent if field.db and field.type.startswith('reference') and \ field.type.find('.') < 0 and field.type[10:] in field.db.tables: referenced = field.db[field.type[10:]] return _repr_ref(referenced) elif field.db and field.type.startswith('list:reference') and \ field.type.find('.') < 0 and field.type[15:] in field.db.tables: referenced = field.db[field.type[15:]] return _repr_ref_list(referenced) return field.represent def varquote_aux(name, quotestr='%s'): return name if REGEX_W.match(name) else quotestr % name def uuid2int(uuidv): return uuid.UUID(uuidv).int def int2uuid(n): return str(uuid.UUID(int=n)) # Geodal utils def geoPoint(x, y): return "POINT (%f %f)" % (x, y) def geoLine(*line): return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line) def geoPolygon(*line): return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line) # upload utils def attempt_upload(table, fields): for fieldname in table._upload_fieldnames & set(fields): value = fields[fieldname] if not (value is None or isinstance(value, string_types)): if not PY2 and isinstance(value, bytes): continue if hasattr(value, 'file') and hasattr(value, 'filename'): new_name = table[fieldname].store( value.file, filename=value.filename) elif isinstance(value, dict): if 'data' in value and 'filename' in value: stream = BytesIO(to_bytes(value['data'])) new_name = table[fieldname].store( stream, filename=value['filename']) else: new_name = None elif hasattr(value, 'read') and hasattr(value, 'name'): new_name = table[fieldname].store(value, filename=value.name) else: raise RuntimeError("Unable to handle upload") fields[fieldname] = new_name def attempt_upload_on_insert(table): def wrapped(fields): return attempt_upload(table, fields) return wrapped def attempt_upload_on_update(table): def wrapped(dbset, fields): return attempt_upload(table, fields) return wrapped def delete_uploaded_files(dbset, upload_fields=None): table = dbset.db._adapter.tables(dbset.query).popitem()[1] # ## mind uploadfield==True means file is not in DB if upload_fields: fields = list(upload_fields) # Explicitly add compute upload fields (ex: thumbnail) fields += [f for f in table.fields if table[f].compute is not None] else: fields = table.fields fields = [ f for f in fields if table[f].type == 'upload' and table[f].uploadfield == True and table[f].autodelete ] if not fields: return False for record in dbset.select(*[table[f] for f in fields]): for fieldname in fields: field = table[fieldname] oldname = record.get(fieldname, None) if not oldname: continue if upload_fields and fieldname in upload_fields and \ oldname == upload_fields[fieldname]: continue if field.custom_delete: field.custom_delete(oldname) else: uploadfolder = field.uploadfolder if not uploadfolder: uploadfolder = pjoin( dbset.db._adapter.folder, '..', 'uploads') if field.uploadseparate: items = oldname.split('.') uploadfolder = pjoin( uploadfolder, "%s.%s" % (items[0], items[1]), items[2][:2]) oldpath = pjoin(uploadfolder, oldname) if exists(oldpath): os.unlink(oldpath) return False