test.py

#!/usr/bin/env python

# Automated script to run the pspautotests test suite in PPSSPP.

import sys
import os
import subprocess
import threading
import glob
import platform


PPSSPP_EXECUTABLES = [
  # Windows. The machine's own architecture comes first: an x64 build runs on Windows-on-ARM too,
  # under emulation, so looking for it first would quietly test the emulated build instead.
  "Windows\\Debug\\PPSSPPHeadless.exe",
  "Windows\\Release\\PPSSPPHeadless.exe",
] + ([
  "Windows\\ARM64\\Debug\\PPSSPPHeadless.exe",
  "Windows\\ARM64\\Release\\PPSSPPHeadless.exe",
] if platform.machine().lower() in ("arm64", "aarch64") else []) + [
  "Windows\\x64\\Debug\\PPSSPPHeadless.exe",
  "Windows\\x64\\Release\\PPSSPPHeadless.exe",
  "build*/PPSSPPHeadless.exe",
  "./PPSSPPHeadless.exe",
  # Mac
  "build*/Debug/PPSSPPHeadless",
  "build*/Release/PPSSPPHeadless",
  "build*/RelWithDebInfo/PPSSPPHeadless",
  "build*/MinSizeRel/PPSSPPHeadless",
  # Linux
  "build*/PPSSPPHeadless",
  "./PPSSPPHeadless",
  # CI
  "ppsspp/PPSSPPHeadless",
  "ppsspp\\PPSSPPHeadless.exe",
]

PPSSPP_EXE = None
TEST_ROOT = "pspautotests/tests/"
TIMEOUT = 5

# The slower CPU backends need a longer wall clock on the CPU-heavy tests - the interpreter runs
# misc/deadbeef in about 3s. Scale the timeout per backend rather than raising it for everyone, so
# a genuine hang under the JIT is still caught in five seconds.
CPU_TIMEOUTS = {
  'interpreter': 20,
  'ir': 10,
  'jit': 5,
  'jit-ir': 5,
}

DEBUG_TIMEOUT_SCALE = 3

class Command(object):
  def __init__(self, cmd, data = None):
    self.cmd = cmd
    self.data = data
    self.process = None
    self.output = None
    self.timeout = False

  def run(self, timeout):
    def target():
      self.process = subprocess.Popen(self.cmd, stdin=subprocess.PIPE, stdout=sys.stdout, stderr=subprocess.STDOUT)
      self.process.stdin.write(self.data.encode('utf-8'))
      self.process.stdin.close()
      self.process.communicate()

    thread = threading.Thread(target=target)
    thread.start()

    thread.join(timeout)
    if thread.is_alive():
      self.timeout = True
      if sys.version_info < (2, 6):
        os.kill(self.process.pid, signal.SIGKILL)
      else:
        self.process.terminate()
      thread.join()

    return self.process.returncode

# Test names are the C files without the .c extension.
# These have worked and should keep working always - regression tests.
# -g flag runs these.
tests_good = [
  "cpu/cpu_alu/cpu_alu",
  "cpu/cpu_alu/cpu_branch",
  "cpu/cpu_alu/cpu_branch2",
  "cpu/cpu_alu/cpu_div",
  "cpu/vfpu/callout",
  "cpu/vfpu/colors",
  "cpu/vfpu/convert",
  "cpu/vfpu/convert_scaled",
  "cpu/vfpu/minmax",
  "cpu/vfpu/prefix_branch",
  "cpu/vfpu/prefix_ctrl",
  "cpu/vfpu/vbranch",
  "cpu/vfpu/vrnd",
  "cpu/vfpu/overlap",
  "cpu/vfpu/gum",
  "cpu/vfpu/matrix",
  "cpu/vfpu/vavg",
  "cpu/vfpu/exact",
  "cpu/vfpu/vrot",
  "cpu/icache/icache",
  "cpu/lsu/lsu",
  "cpu/lsu/llsc",
  "cpu/fpu/fpu",
  "cpu/fpu/rounding",
  "cpu/fpu/roundmode",
  "cpu/fpu/fpu_branch",

  "audio/atrac/addstreamdata",
  "audio/atrac/atractest",
  "audio/atrac/decode",
  "audio/atrac/getremainframe",
  "audio/atrac/getsoundsample",
  "audio/atrac/ids",
  "audio/atrac/resetpos",
  "audio/atrac/resetting",
  "audio/atrac/replay",
  "audio/atrac/stream",
  "audio/atrac/reset2",
  "audio/atrac/second/resetting",
  "audio/atrac/second/getinfo",
  "audio/atrac/second/needed",
  "audio/atrac/second/setbuffer",
  "audio/atrac/setdata",
  "audio/atrac/sas",
  "audio/atrac/c0mono",
  "audio/audiocodec/basic",
  "audio/audiocodec/at3param",
  "audio/mp3/checkneeded",
  "audio/mp3/getbitrate",
  "audio/mp3/getchannel",
  "audio/mp3/getframenum",
  "audio/mp3/getloopnum",
  "audio/mp3/getmaxoutput",
  "audio/mp3/getmpegversion",
  "audio/mp3/getsamplerate",
  "audio/mp3/getsumdecoded",
  "audio/mp3/infotoadd",
  "audio/mp3/initresource",
  "audio/mp3/mp3test",
  "audio/mp3/notifyadd",
  "audio/mp3/release",
  "audio/mp3/reserve",
  "audio/mp3/setloopnum",
  "audio/mp3/stream",
  "audio/blocking/contend",
  "audio/blocking/channels",
  "audio/blocking/depth",
  "audio/blocking/errors",
  "audio/blocking/overhead",
  "audio/blocking/parked",
  "audio/blocking/oneshot",
  "audio/blocking/restlen",
  "audio/blocking/vaudio",
  "audio/sceaudio/reserve",
  "audio/sceaudio/datalen",
  "audio/output2/changelength",
  "audio/output2/release",
  "audio/output2/reserve",
  "audio/output2/threads",
  "audio/reverb/basic",
  "audio/reverb/volume",
  "audio/sascore/sascore",
  "audio/sascore/adsrcurve",
  "audio/sascore/getheight",
  "audio/sascore/keyoff",
  "audio/sascore/keyon",
  "audio/sascore/noise",
  "audio/sascore/outputmode",
  "audio/sascore/pause",
  "audio/sascore/pcm",
  "audio/sascore/pitch",
  "audio/sascore/vag",
  "ctrl/ctrl",
  "ctrl/idle/idle",
  "ctrl/sampling/sampling",
  "ctrl/sampling2/sampling2",
  "ctrl/vblank",
  "display/display",
  "display/hcount",
  "display/vblankmulti",
  "display/isstate",
  "display/setframebuf",
  "display/setmode",
  "dmac/dmactest",
  "font/altcharcode",
  "font/charglyphimage",
  "font/charglyphimageclip",
  "font/charimagerect",
  "font/find",
  "font/fontinfo",
  "font/fontinfobyindex",
  "font/fontlist",
  "font/optimum",
  "font/resolution",
  "font/shadowimagerect",
  "gpu/bounding/count",
  "gpu/bounding/planes",
  "gpu/bounding/vertexaddr",
  "gpu/bounding/viewport",
  "gpu/callbacks/ge_callbacks",
  "gpu/clipping/homogeneous",
  "gpu/clut/address",
  "gpu/clut/masks",
  "gpu/clut/offset",
  "gpu/clut/shifts",
  "gpu/commands/basic",
  "gpu/commands/blend",
  "gpu/commands/blend565",
  "gpu/commands/blocktransfer",
  "gpu/commands/cull",
  "gpu/commands/fog",
  "gpu/commands/material",
  "gpu/complex/complex",
  "gpu/displaylist/alignment",
  "gpu/dither/dither",
  "gpu/filtering/mipmaplinear",
  "gpu/ge/break",
  "gpu/ge/breakwait",
  "gpu/ge/callbackstate",
  "gpu/ge/context",
  "gpu/ge/edram",
  "gpu/ge/enqueueparam",
  "gpu/ge/intrsuspend",
  "gpu/ge/queue",
  "gpu/ge/queue2",
  "gpu/lighting/shademap",
  "gpu/lighting/specular",
  "gpu/primitives/indices",
  "gpu/primitives/invalidprim",
  "gpu/primitives/points",
  "gpu/primitives/rectangles",
  "gpu/primitives/trianglefan",
  "gpu/primitives/trianglestrip",
  "gpu/primitives/triangles",
  "gpu/rendertarget/copy",
  "gpu/rendertarget/depal",
  "gpu/signals/pause",
  "gpu/signals/pause2",
  "gpu/signals/suspend",
  "gpu/signals/sync",
  "gpu/texcolors/dxt1",
  "gpu/texcolors/dxt3",
  "gpu/texcolors/dxt5",
  "gpu/texcolors/rgb565",
  "gpu/texcolors/rgba4444",
  "gpu/texcolors/rgba5551",
  "gpu/texfunc/add",
  "gpu/texfunc/blend",
  "gpu/texfunc/decal",
  "gpu/texfunc/modulate",
  "gpu/texfunc/replace",
  "gpu/textures/mipmap",
  "gpu/textures/rotate",
  "gpu/transfer/invalid",
  "gpu/transfer/mirrors",
  "gpu/transfer/overlap",
  "gpu/vertices/colors",
  "gpu/vertices/morph",
  # "gpu/vertices/texcoords",  #  See issue #19093
  "hash/hash",
  "hash/md5ctx",
  "hash/mt19937ctx",
  "hash/sha1ctx",
  "hle/check_not_used_uids",
  "intr/intr",
  "intr/mfic",
  "intr/enablesub",
  "intr/registersub",
  "intr/releasesub",
  "intr/suspended",
  "intr/vblank/vblank",
  "intr/delays/delays",
  "io/cwd/cwd",
  "io/file/rename",
  "io/directory/directory",
  "io/stat/stat",
  "io/stat/readonly",
  "io/open/badparent",
  "jpeg/create",
  "jpeg/delete",
  "jpeg/finish",
  "jpeg/init",
  "loader/bss/bss",
  "malloc/malloc",
  "misc/dcache",
  "misc/deadbeef",
  "modules/unresolved/unresolved",
  "misc/libc",
  "misc/sdkver",
  "misc/testgp",
  "misc/timeconv",
  "misc/reg",
  "mstick/mstick",
  "power/cpu",
  "power/power",
  "power/freq",
  "power/volatile/lock",
  "power/volatile/trylock",
  "power/volatile/unlock",
  "rtc/rtc",
  "rtc/arithmetic",
  "rtc/lookup",
  "rtc/convert",
  "string/string",
  "sysmem/freesize",
  "sysmem/memblock",
  "sysmem/sysmem",
  "sysmem/partitions",
  "sysmem/kernel/partitions",
  "sysmem/kernel/heap",
  "sysmem/volatile",
  "threads/alarm/alarm",
  "threads/alarm/cancel/cancel",
  "threads/alarm/refer/refer",
  "threads/alarm/set/set",
  "threads/callbacks/afterwait",
  "threads/callbacks/callbacks",
  "threads/callbacks/cancel",
  "threads/callbacks/cbtimeout",
  "threads/callbacks/check",
  "threads/callbacks/count",
  "threads/callbacks/create",
  "threads/callbacks/delete",
  "threads/callbacks/delivery",
  "threads/callbacks/exit",
  "threads/callbacks/intrnotify",
  "threads/callbacks/nested",
  "threads/callbacks/notify",
  "threads/callbacks/otherthread",
  "threads/callbacks/recursion",
  "threads/callbacks/refer",
  "threads/callbacks/waittypes",
  "threads/events/events",
  "threads/events/cancel/cancel",
  "threads/events/clear/clear",
  "threads/events/create/create",
  "threads/events/delete/delete",
  "threads/events/poll/poll",
  "threads/events/refer/refer",
  "threads/events/set/set",
  "threads/events/wait/wait",
  "threads/fpl/fpl",
  "threads/fpl/allocate",
  "threads/fpl/cancel",
  "threads/fpl/create",
  "threads/fpl/delete",
  "threads/fpl/free",
  "threads/fpl/priority",
  "threads/fpl/refer",
  "threads/fpl/tryallocate",
  "threads/k0/k0",
  "threads/lwmutex/callbacks",
  "threads/lwmutex/create",
  "threads/lwmutex/delete",
  "threads/lwmutex/lock",
  "threads/lwmutex/priority",
  "threads/lwmutex/refer",
  "threads/lwmutex/try",
  "threads/lwmutex/try600",
  "threads/lwmutex/unlock",
  "threads/mbx/mbx",
  "threads/mbx/cancel/cancel",
  "threads/mbx/create/create",
  "threads/mbx/delete/delete",
  "threads/mbx/poll/poll",
  "threads/mbx/priority/priority",
  "threads/mbx/receive/receive",
  "threads/mbx/refer/refer",
  "threads/mbx/send/send",
  "threads/msgpipe/msgpipe",
  "threads/msgpipe/cancel",
  "threads/msgpipe/create",
  "threads/msgpipe/data",
  "threads/msgpipe/delete",
  "threads/msgpipe/receive",
  "threads/msgpipe/refer",
  "threads/msgpipe/send",
  "threads/msgpipe/tryreceive",
  "threads/msgpipe/trysend",
  "threads/mutex/cancel",
  "threads/mutex/create",
  "threads/mutex/delete",
  "threads/mutex/lock",
  "threads/mutex/mutex",
  "threads/mutex/priority",
  "threads/mutex/refer",
  "threads/mutex/try",
  "threads/mutex/unlock",
  "threads/mutex/unlock2",
  "threads/scheduling/dispatch",
  "threads/callbacks/combos",
  "threads/scheduling/delayzero",
  "threads/scheduling/dispatchwake",
  "threads/scheduling/mutexhandoff",
  "threads/scheduling/vblankwake",
  "threads/scheduling/readyqueue",
  "threads/scheduling/waittimeouts",
  "threads/semaphores/semaphores",
  "threads/semaphores/cancel",
  "threads/semaphores/create",
  "threads/semaphores/delete",
  "threads/semaphores/fifo",
  "threads/semaphores/poll",
  "threads/semaphores/priority",
  "threads/semaphores/refer",
  "threads/semaphores/signal",
  "threads/semaphores/wait",
  "threads/threads/change",
  "threads/threads/exitstatus",
  "threads/threads/extend",
  "threads/threads/refer",
  "threads/threads/release",
  "threads/threads/rotate",
  "threads/threads/stackfree",
  "threads/threads/start",
  "threads/threads/suspend",
  "threads/threads/terminate",
  "threads/threads/termsuspended",
  "threads/threads/threadend",
  "threads/threads/threadmanidlist",
  "threads/threads/threadmanidtype",
  "threads/threads/threads",
  "threads/tls/allocate",
  "threads/tls/create",
  "threads/tls/partition",
  "threads/tls/kernel/partition",
  "threads/tls/delete",
  "threads/tls/get",
  "threads/tls/free",
  "threads/tls/priority",
  "threads/tls/refer",
  "threads/tls/timeout",
  "threads/vpl/allocate",
  "threads/vpl/cancel",
  "threads/vpl/create",
  "threads/vpl/delete",
  "threads/vpl/fifo",
  "threads/vpl/free",
  "threads/vpl/order",
  "threads/vpl/priority",
  "threads/vpl/refer",
  "threads/vpl/try",
  "threads/vpl/vpl",
  "threads/vtimers/vtimer",
  "threads/vtimers/cancelhandler",
  "threads/vtimers/create",
  "threads/vtimers/delete",
  "threads/vtimers/getbase",
  "threads/vtimers/gettime",
  "threads/vtimers/interrupt",
  "threads/vtimers/refer",
  "threads/vtimers/sethandler",
  "threads/vtimers/settime",
  "threads/vtimers/start",
  "threads/vtimers/stop",
  "threads/wakeup/wakeup",
  "utility/dialog/abort",
  "utility/dialog/htmlviewer",
  "utility/dialog/priority",
  "utility/dialog/sizes",
  "utility/dialog/status",
  "utility/msgdialog/abort",
  "utility/savedata/autosave",
  "utility/savedata/shutdownstatus",
  "utility/savedata/filelist",
  "utility/savedata/getsize",
  "utility/savedata/makedata",
  "utility/systemparam/systemparam",
  "umd/api/api",
  "umd/callbacks/umd",
  "umd/wait/wait",
  "umd/register",
  "video/mpeg/ringbuffer/avail",
  "video/mpeg/ringbuffer/construct",
  "video/mpeg/ringbuffer/destruct",
  "video/mpeg/ringbuffer/memsize",
  "video/mpeg/ringbuffer/packnum",
  "video/psmfplayer/break",
  "video/psmfplayer/create",
  "video/psmfplayer/delete",
  "video/psmfplayer/getaudiodata",
  "video/psmfplayer/getaudiooutsize",
  "video/psmfplayer/getcurrentpts",
  "video/psmfplayer/getcurrentstatus",
  "video/psmfplayer/getcurrentstream",
  "video/psmfplayer/getpsmfinfo",
  "video/psmfplayer/releasepsmf",
  "video/psmfplayer/selectspecific",
  "video/psmfplayer/setpsmf",
  "video/psmfplayer/settempbuf",
  "video/psmfplayer/stop",
]

# Broken tests
# -b flag runs these.

# Tests that don't pass yet on an architecture we can only reach through emulation. Pass
# --known-failures=<arch> to drop them from the run, so CI can still catch anything *new* breaking
# while these stay outstanding. Keep a reason next to each one, and delete entries as they're fixed
# rather than letting the list rot.
known_failures = {
  "riscv64": [
    # No flush-to-zero: the ISA has no control for it, so a denormal result survives where the
    # PSP would have flushed it. Everything else in this test passes.
    "cpu/fpu/fpu",
    # The ISA returns the canonical NaN (0x7fc00000) from every operation, never the operand's
    # NaN, so a negative or signaling NaN input loses its sign and payload. Everything else passes.
    "cpu/fpu/roundmode",
  ],
  "loongarch64": [
    "cpu/fpu/fpu",
  ],
}

tests_next = [
# These are the next tests up for fixing. These run by default.
  "cpu/fpu/fcr",
  "cpu/vfpu/prefix_consume",  # see the pspautotests commit for what differs per core
  "cpu/vfpu/prefix_sat",
  "cpu/vfpu/prefix_unpack",  # an invalid swizzle replays an earlier prefixed value, not emulated
  "cpu/vfpu/vbranch_hazard",  # VFPU pipeline latencies, which a compiler pads for; not emulated
  "cpu/vfpu/minmax_tie",  # vmin/vmax return the second operand on a -0/+0 tie; the IR path returns the first
  "cpu/vfpu/minmax_zero",  # signed zero and denormals in vmin/vmax
  "cpu/vfpu/specials",  # vcmp on denormals, NaN canonicalization and denormal flush in vbfy/vocp/vavg/vfad/vsocp
  "cpu/vfpu/overlap_vcrsp",  # vcrsp overlapping its source, which the assembler refuses; the hardware doesn't read-before-write
  "cpu/fpu/fpu_branch_hazard",  # a bc1x right after c.xx.s sees the old condition; the compiler pads for it, not emulated
  "cpu/fpu/fpu_nan",  # 0/0 and inf-inf give 0x7fc00000; x86 hosts make 0xffc00000, and a check per op isn't worth it
  "cpu/lsu/cacheop",  # the data cache is write-back and the uncached mirror shows it; not emulated
  "cpu/vfpu/prefixes",
  "cpu/vfpu/vector",
  "cpu/vfpu/vregs",
  "audio/sceaudio/output",
  "audio/sascore/setadsr",
  "audio/mp3/init",
  "audio/output2/frequency",
  "audio/output2/rest",
  "ccc/convertstring",
  "font/fonttest",
  "font/charinfo",
  "font/newlib",
  "font/open",
  "font/openfile",
  "font/openmem",
  "font/shadowglyphimage",
  "font/shadowglyphimageclip",
  "font/shadowinfo",
  "gpu/clipping/guardband",
  "gpu/commands/light",
  "gpu/depth/precision",
  "gpu/displaylist/state",
  "gpu/filtering/linear",
  "gpu/filtering/nearest",
  "gpu/filtering/precisionlinear2d",
  "gpu/filtering/precisionlinear3d",
  "gpu/filtering/precisionnearest2d",
  "gpu/filtering/precisionnearest3d",
  "gpu/ge/edramswizzle",
  "gpu/ge/get",
  "gpu/primitives/bezier",
  "gpu/primitives/continue",
  "gpu/primitives/immediate",
  "gpu/primitives/indices32",
  "gpu/primitives/lines",
  "gpu/primitives/linestrip",
  "gpu/primitives/spline",
  "gpu/reflection/reflection",
  "gpu/rendertarget/rendertarget",
  "gpu/signals/continue",
  # Old SDK: a stall update from inside a SUSPEND handler is remembered, but not applied to the GE. See docs/sceGe.md.
  "gpu/signals/handlercalls",
  "gpu/signals/jumps",
  "gpu/signals/simple",
  "gpu/simple/simple",
  "gpu/texmtx/normals",
  "gpu/texmtx/prims",
  "gpu/texmtx/source",
  "gpu/texmtx/uvs",
  "gpu/textures/size",
  "gpu/triangle/triangle",
  "intr/waits",
  "sysmem/kernel/heapgrow",
  "io/file/file",
  "io/io/io",
  "io/iodrv/iodrv",
  "io/shortname/shortname",
  "io/open/tty0",
  "jpeg/csc",
  "jpeg/decode",
  "jpeg/decodes",
  "jpeg/decodeycbcr",
  "jpeg/decodeycbcrs",
  "jpeg/getoutputinfo",
  "jpeg/mjpegcsc",
  # Doesn't work on a PSP for security reasons, hangs in PPSSPP currently.
  # Commented out to make tests run much faster.
  #"modules/loadexec/loader",
  "net/http/http",
  "net/primary/ether",
  "sysmem/partition",
  # These two mbx tests only appeared to work because they papered over bugs 


  "threads/scheduling/scheduling",
  "threads/threads/create",
  "threads/tls/memory",
  "umd/io/umd_io",
  "umd/raw_access/raw_access",
  "utility/msgdialog/dialog",
  "utility/savedata/idlist",
  # These tests appear to be broken and just hang.
  #"utility/savedata/deletebroken",
  #"utility/savedata/deletedata",
  #"utility/savedata/deleteemptyfilename",
  #"utility/savedata/loadbroken",
  #"utility/savedata/loaddata",
  #"utility/savedata/loademptyfilename",
  #"utility/savedata/saveemptyfilename",
  "utility/savedata/secureversion",
  "utility/savedata/sizes",
  "video/mpeg/basic",
  "video/pmf/pmf",
  "video/pmf_simple/pmf_simple",
  "video/psmfplayer/basic",
  "video/psmfplayer/configplayer",
  "video/psmfplayer/getvideodata",
  "video/psmfplayer/playmode",
  "video/psmfplayer/selectstream",
  "video/psmfplayer/setpsmfoffset",
  "video/psmfplayer/start",
  "video/psmfplayer/update",
]


# These are the tests we ignore (not important, or impossible to run)
tests_ignored = [
  "kirk/kirk",
  "me/me",
]



def init():
  global PPSSPP_EXE, TEST_ROOT
  if not os.path.exists("pspautotests"):
    if os.path.exists(os.path.dirname(__file__) + "/pspautotests"):
      TEST_ROOT = os.path.dirname(__file__) + "/pspautotests/tests/";
    else:
      print("Please run git submodule init; git submodule update;")
      sys.exit(1)

  if not os.path.exists(TEST_ROOT + "cpu/cpu_alu/cpu_alu.prx"):
    print("Please install the pspsdk and run make in common/ and in all the tests")
    print("(checked for existence of cpu/cpu_alu/cpu_alu.prx)")
    sys.exit(1)

  possible_exes = [glob.glob(f) for f in PPSSPP_EXECUTABLES]
  possible_exes = [x for sublist in possible_exes for x in sublist]
  existing = filter(os.path.exists, possible_exes)
  if existing:
    PPSSPP_EXE = max((os.path.getmtime(f), f) for f in existing)[1]
  else:
    PPSSPP_EXE = None

  if not PPSSPP_EXE:
    print("PPSSPPHeadless executable missing, please build one.")
    sys.exit(1)

def cpu_backend(args):
  # Which backend headless will end up on, given the args we hand through to it. Headless defaults
  # to the JIT, and a later flag overrides an earlier one, like its own parsing in Core/CmdLine.cpp.
  short_flags = {'-i': 'interpreter', '-r': 'ir', '-j': 'jit', '-J': 'jit-ir'}
  backend = 'jit'
  for arg in args:
    if arg in short_flags:
      backend = short_flags[arg]
    elif arg.startswith('--cpu='):
      backend = arg[len('--cpu='):]
  return backend

def run_tests(test_list, args):
  global PPSSPP_EXE, TIMEOUT
  returncode = 0
  timeout = CPU_TIMEOUTS.get(cpu_backend(args), TIMEOUT)
  # Debug builds are several times slower, enough for the heavier tests to brush the limit.
  if 'Debug' in os.path.normpath(PPSSPP_EXE).split(os.sep):
    timeout *= DEBUG_TIMEOUT_SCALE

  test_filenames = []
  for test in test_list:
    # Try prx first
    elf_filename = TEST_ROOT + test + ".prx"
    if not os.path.exists(elf_filename):
      print("WARNING: no prx, trying elf")
      elf_filename = TEST_ROOT + test + ".elf"

    test_filenames.append(elf_filename)

  if len(test_filenames):
    # TODO: Maybe --compare should detect --graphics?
    cmdline = [PPSSPP_EXE, '--root', TEST_ROOT + '../', '--compare', '--timeout-wall=' + str(timeout), '@-']
    cmdline.extend([i for i in args if i not in ['-g', '-m', '-b']])

    c = Command(cmdline, '\n'.join(test_filenames))
    returncode = c.run(timeout * len(test_filenames))

    print("Ran " + ' '.join(cmdline))

  return returncode

def main():
  init()
  tests = []
  args = []
  teamcity = False
  skip_arch = None
  for arg in sys.argv[1:]:
    if arg == '--teamcity':
      args.append(arg)
      teamcity = True
    elif arg.startswith('--known-failures='):
      # Ours, not headless's - don't pass it through.
      skip_arch = arg[len('--known-failures='):]
      if skip_arch not in known_failures:
        print("Unknown architecture for --known-failures: " + skip_arch)
        sys.exit(1)
    elif arg[0] == '-':
      args.append(arg)
    else:
      tests.append(arg)

  if not tests:
    if '-g' in args:
      tests = tests_good
    elif '-b' in args:
      tests = tests_next
    else:
      tests = tests_next + tests_good
  elif '-m' in args and '-g' in args:
    tests = [i for i in tests_good if i.startswith(tests[0])]
  elif '-m' in args and '-b' in args:
    tests = [i for i in tests_next if i.startswith(tests[0])]
  elif '-m' in args:
    tests = [i for i in tests_next + tests_good if i.startswith(tests[0])]

  if skip_arch:
    skipped = [t for t in tests if t in known_failures[skip_arch]]
    tests = [t for t in tests if t not in known_failures[skip_arch]]
    if skipped:
      print("Skipping %d known failures on %s: %s" % (len(skipped), skip_arch, ", ".join(skipped)))

  returncode = run_tests(tests, args)
  if teamcity:
    return 0
  return returncode

exit(main())