{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/secqa-a-concise-question-answering-dataset","title":"SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security","arxiv_id":"2312.15838","date":"2023-12-26","proceeding":null,"authors":["Zefang Liu"],"abstract":"In this paper, we introduce SecQA, a novel dataset tailored for evaluating the performance of Large Language Models (LLMs) in the domain of computer security. Utilizing multiple-choice questions generated by GPT-4 based on the \"Computer Systems Security: Planning for Success\" textbook, SecQA aims to assess LLMs' understanding and application of security principles. We detail the structure and intent of SecQA, which includes two versions of increasing complexity, to provide a concise evaluation across various difficulty levels. Additionally, we present an extensive evaluation of prominent LLMs, including GPT-3.5-Turbo, GPT-4, Llama-2, Vicuna, Mistral, and Zephyr models, using both 0-shot and 5-shot learning settings. Our results, encapsulated in the SecQA v1 and v2 datasets, highlight the varying capabilities and limitations of these models in the computer security context. This study not only offers insights into the current state of LLMs in understanding security-related content but also establishes SecQA as a benchmark for future advancements in this critical research area.","url_abs":"https://arxiv.org/abs/2312.15838v1","url_pdf":"https://arxiv.org/pdf/2312.15838v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"secqa-a-concise-question-answering-dataset","repo_url":"https://github.com/zefang-liu/lm-evaluation-harness","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"computer-security","task_name":"Computer Security"},{"task_slug":"multiple-choice","task_name":"Multiple-choice"},{"task_slug":"question-answering","task_name":"Question Answering"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[{"slug":"secqa","name":"SecQA","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.15838","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.15838"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zefang-liu/lm-evaluation-harness","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"57e732507bad40db","entry":"escaped_split","repo":"zefang-liu/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/utils.py","file_url":"https://github.com/zefang-liu/lm-evaluation-harness/blob/HEAD/lm_eval/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"57e732507bad40db"}},{"code_sha256_prefix":"53354f082d9addc6","entry":"get_result","repo":"zefang-liu/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/models/gguf.py","file_url":"https://github.com/zefang-liu/lm-evaluation-harness/blob/HEAD/lm_eval/models/gguf.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"53354f082d9addc6"}},{"code_sha256_prefix":"8cdc4b3572832c60","entry":"get_result","repo":"zefang-liu/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/models/openai_completions.py","file_url":"https://github.com/zefang-liu/lm-evaluation-harness/blob/HEAD/lm_eval/models/openai_completions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8cdc4b3572832c60"}},{"code_sha256_prefix":"0ee5d4d20cf990bd","entry":"handle_arg_string","repo":"zefang-liu/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/utils.py","file_url":"https://github.com/zefang-liu/lm-evaluation-harness/blob/HEAD/lm_eval/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0ee5d4d20cf990bd"}},{"code_sha256_prefix":"20a7cc804eb22661","entry":"hash_args","repo":"zefang-liu/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/api/model.py","file_url":"https://github.com/zefang-liu/lm-evaluation-harness/blob/HEAD/lm_eval/api/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20a7cc804eb22661"}},{"code_sha256_prefix":"dfed6e9291316231","entry":"simple_parse_args_string","repo":"zefang-liu/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/utils.py","file_url":"https://github.com/zefang-liu/lm-evaluation-harness/blob/HEAD/lm_eval/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dfed6e9291316231"}},{"code_sha256_prefix":"8cb5bae50022f151","entry":"run_inference_one_model","repo":"zefang-liu/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/models/vllm_causallms.py","file_url":"https://github.com/zefang-liu/lm-evaluation-harness/blob/HEAD/lm_eval/models/vllm_causallms.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8cb5bae50022f151"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}