args = set(args)

if not (args & {'standby', 'no_standby'}):

standby = 'standby' if GlobalVars.standby_mode else 'no_standby'

args.add(standby)

with open("exit.txt", "w", encoding="utf-8") as f:

print("\n".join(args), file=f)

log('debug', 'Exiting with args: {}'.format(', '.join(args) or 'None'))

# Flush any buffered queue timing data

import datahandling # this must not be a top-level import in order to avoid a circular import

datahandling.flush_queue_timings_data()

datahandling.store_post_scan_stats()

datahandling.store_recently_scanned_posts()

# We have to use '_exit' here, because 'sys.exit' only exits the current

# thread (not the current process). Unfortunately, this results in

# 'atexit' handlers not being called. All exit calls in SmokeDetector go

# through this function, so any necessary cleanup can happen here (though

# keep in mind that this function isn't called when terminating due to a

# Ctrl-C or other signal).

DB_FILE = "errorLogs.db"

# SQLite threading limitation !?!?!?

db = sqlite3.connect(DB_FILE)

if db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='error_logs'").fetchone() is None:

# Table 'error_logs' doesn't exist

try:

db.execute("CREATE TABLE error_logs (time REAL PRIMARY KEY ASC, classname TEXT, message TEXT,"

" traceback TEXT)")

db.commit()

except (sqlite3.OperationalError):

# In CI testing, it's possible for the table to be created in a different thread between when

# we first test for the table's existence and when we try to create the table.

if db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='error_logs'").fetchone() is None:

# Table 'error_logs' still doesn't exist

raise

db.close()

db_conns = {}

@classmethod

def get_db(cls):

thread_id = threading.get_ident()

if thread_id not in cls.db_conns:

cls.db_conns[thread_id] = sqlite3.connect(cls.DB_FILE)

return cls.db_conns[thread_id]

@classmethod

def add_current_exception(cls):

now = datetime.utcnow()

exctype, value, traceback_or_message = sys.exc_info()

tr = get_traceback_from_traceback_or_message(traceback_or_message)

cls.add(now.timestamp(), exctype.__name__, str(value), tr)

@classmethod

def add(cls, time, classname, message, traceback):

classname = redact_passwords(classname)

message = redact_passwords(message)

traceback = redact_passwords(traceback)

db = cls.get_db()

db.execute("INSERT INTO error_logs VALUES (?, ?, ?, ?)",

(time, classname, message, traceback))

db.commit()

@classmethod

def fetch_last(cls, n):

db = cls.get_db()

cursor = db.execute("SELECT * FROM error_logs ORDER BY time DESC LIMIT ?", (int(n),))

data = cursor.fetchall()

return data

@classmethod

def truncate(cls, n=100):

"""

Truncate the DB and keep only N latest exceptions

"""

db = cls.get_db()

cursor = db.execute(

"DELETE FROM error_logs WHERE time IN "

"(SELECT time FROM error_logs ORDER BY time DESC LIMIT -1 OFFSET ?)", (int(n),))

db.commit()

data = cursor.fetchall()

s = s.lower()

if s.endswith("so"):

s = s[:-2] + "stackoverflow.com"

elif s.endswith("se"):

s = s[:-2] + "stackexchange.com"

elif s.endswith("su"):

s = s[:-2] + "superuser.com"

elif s.endswith("sf"):

s = s[:-2] + "serverfault.com"

elif s.endswith("au"):

s = s[:-2] + "askubuntu.com"

if redact_str:

return text.replace(redact_str, replace_with) \

.replace(quote(redact_str), replace_with) \

.replace(quote_plus(redact_str), replace_with)

value = str(value)

# Generic redaction of URLs with http, https, and ftp schemes

value = regex.sub(r"((?:https?|ftp):\/\/)[^@:\/]*:[^@:\/]*(?=@)", r"\1[REDACTED URL USERNAME AND PASSWORD]", value)

# In case these are somewhere else.

value = redact_text(value, GlobalVars.github_password, "[GITHUB PASSWORD REDACTED]")

value = redact_text(value, GlobalVars.github_access_token, "[GITHUB ACCESS TOKEN REDACTED]")

value = redact_text(value, GlobalVars.chatexchange_p, "[CHAT PASSWORD REDACTED]")

value = redact_text(value, GlobalVars.metasmoke_key, "[METASMOKE KEY REDACTED]")

value = redact_text(value, GlobalVars.perspective_key, "[PERSPECTIVE KEY REDACTED]")

levels = {

'debug': [0, 'grey'],

'info': [1, 'cyan'],

'warning': [2, 'yellow'],

'warn': [2, 'yellow'],

'error': [3, 'red']

}

level = levels[log_level][0]

if level < Helpers.min_log_level:

return

color = levels[log_level][1] if log_level in levels else 'white'

log_str = "{} {}".format(colored("[{}]".format(datetime.utcnow().isoformat()[11:-3]),

color, attrs=['bold']),

redact_passwords(" ".join([str(x) for x in args])))

print(log_str, file=sys.stderr)

if level == 3 and not no_exception:

exc_tb = sys.exc_info()[2]

print(redact_passwords("".join(traceback.format_tb(exc_tb))), file=sys.stderr)

if and_file: # Also to file

levels = {

'debug': 0,

'info': 1,

'warning': 2,

'error': 3,

}

if levels[log_level] < Helpers.min_log_level:

return

log_str = redact_passwords("[{}] {}: {}".format(datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),

log_level.upper(), " ".join([str(x) for x in args])))

with open("errorLogs.txt", "a", encoding="utf-8") as f:

if isinstance(traceback_or_message, str):

return traceback_or_message

else:

log_level = 'error' if log_level is None else log_level

now = datetime.utcnow()

tr = get_traceback_from_traceback_or_message(traceback_or_message)

exception_only = ''.join(traceback.format_exception_only(exctype, value)).strip()

logged_msg = "{exception}\n{now} UTC\n{row}\n\n".format(exception=exception_only, now=now, row=tr)

# Redacting passwords happens in log() and ErrorLogs.add().

log(log_level, logged_msg, and_file=and_file)

if prefix:

prefix += '\t'

if postfix:

postfix = '\t' + postfix

current_thread = threading.current_thread()

"apigetpost.py",

"blacklists.py",

"bodyfetcher.py",

"chatcommands.py",

"chatcommunicate.py",

"chatexchange_extension.py",

"datahandling.py",

"deletionwatcher.py",

"editwatcher.py",

"excepthook.py",

"flovis.py",

"gitmanager.py",

"globalvars.py",

"helpers.py",

"metasmoke.py",

"metasmoke_cache.py",

"nocrash.py",

"number_homoglyphs.py",

"parsing.py",

"phone_numbers.py",

"queue_timings.py",

"recently_scanned_posts.py",

"socketscience.py",

"spamhandling.py",

"tasks.py",

"ws.py",

"classes/feedback.py",

"_Git_Windows.py",

"classes/__init__.py",

"classes/_Post.py",

# Before these are made reloadable

"rooms.yml",

result = True

for s in reloadable_modules:

s = s.replace(".py", "") # Relying on our naming convention

try:

# Some reliable approach

importlib.reload(sys.modules[s])

except (KeyError, ImportError):

result = False

orig_url = url

response_code = 301

# We use a valid Firefox User-Agent, because some services don't provide the forward when we tell them the

# request is from Smokedetector (e.g., cloudflare.com, which is used, as of 2024-04-12, by https://t.ly, which

# is one of the domains which we test in our CI testing).

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0'}

for tries in range(depth):

if response_code not in {301, 302, 303, 307, 308}:

break

res = requests.request(request_type, url, headers=headers, stream=True, allow_redirects=False, timeout=timeout)

res.connection.close() # Discard response body for GET requests

response_code = res.status_code

if 'Location' not in res.headers:

# No more redirects, stop

break

url = res.headers['Location']

else:

raise ValueError("Too many redirects ({}) for URL {!r}".format(depth, orig_url))

bl_files = glob('bad_*.txt') + glob('blacklisted_*.txt') + glob('watched_*.txt')

seen = dict()

errors = []

city_list = ['test']

regex.cache_all(False)

previous_bl_file = ''

for bl_file in bl_files:

print('bl_file:', bl_file)

with open(bl_file, 'r', encoding="utf-8") as lines:

try:

previous_line = ''

previous_line_number = -1

for lineno, line in enumerate(lines, 1):

if line.endswith('\r\n'):

errors.append('{0}:{1}:DOS line ending'.format(bl_file, lineno))

elif not line.endswith('\n'):

errors.append('{0}:{1}:No newline'.format(bl_file, lineno))

elif line == '\n':

errors.append('{0}:{1}:Empty line'.format(bl_file, lineno))

elif bl_file.startswith('watched_'):

line = line.split('\t')[2]

if 'numbers' not in bl_file:

try:

regex.compile(line, regex.UNICODE, city=city_list, ignore_unused=True)

except Exception:

(exctype, value, traceback_or_message) = sys.exc_info()

exception_only = ''.join(traceback.format_exception_only(exctype, value)).strip()

errors.append("{0}:{1}:Regex fails to compile:r'''{2}''':{3}".format(bl_file, lineno,

line.rstrip('\n'),

exception_only))

line = pcre_comment.sub("", line)

if line in seen:

errors.append('{0}:{1}:Duplicate entry {2} (also {3})'.format(

bl_file, lineno, line.rstrip('\n'), seen[line]))

else:

seen[line] = '{0}:{1}'.format(bl_file, lineno)

previous_line = line

previous_line_number = lineno

previous_bl_file = bl_file

except Exception:

# Some output to helps localize the problem

if previous_line_number == -1:

print('Exception raised when started reading: {}'.format(bl_file))

print('End of previous file: line {} of {}:{}'.format(lineno, previous_bl_file, line))

else:

print('previous: line {} of {}:{}'.format(previous_line_number, previous_bl_file, previous_line))

print('line {} of {}:{}'.format(lineno, bl_file, line))

raise

regex.cache_all(True)

"""

Split a list into chunks.

"""

source_text = ""

if source:

source_text = " from {}".format(source)

t = Thread(name="bodyfetcher post enqueuing: {}/{}{}".format(hostname, question_id, source_text),

target=GlobalVars.bodyfetcher.add_to_queue,

args=(hostname, question_id, should_check_site, source))

if type(old_info) is dict:

# This is for recently scanned posts, which pass the recently scanned posts entry here,

# which is a dict.

old_is_spam = old_info.get('is_spam', None)

old_reasons = old_info.get('reasons', None)

old_why = old_info.get('why', None)

elif type(old_info) is tuple:

old_is_spam, old_reasons, old_why = old_info

if not old_is_spam:

return new_info

new_is_spam, new_reasons, new_why = new_info

if new_is_spam:

return new_info

if type(new_reasons) is tuple:

# The scan was actually spam, but was declared non-spam for some reason external to the content.

# For example, that it was recently reported.

actual_new_reasons, actual_new_why = new_reasons

else:

# The new results did not actually indicate it was spam.

return new_info

if match_ignore is not None and new_why not in match_ignore:

# We only want it to be considered spam if ignored for specified reasons.

return new_info

ignored_reasons_set = set()

if ignored_reasons:

if ignored_reasons is True:

ignored_reasons_set = CONVERT_NEW_SCAN_TO_SPAM_DEFAULT_IGNORED_REASONS

else:

ignored_reasons_set = set(ignored_reasons)

actual_new_reasons_set = set(actual_new_reasons) - ignored_reasons_set

old_reasons_set = set(old_reasons) - ignored_reasons_set

if len(actual_new_reasons_set) > len(old_reasons_set) or not actual_new_reasons_set.issubset(old_reasons_set):

# There are new reasons the post would have been reported

return (True, actual_new_reasons, actual_new_why)