ProgrammingBasicScorer/Scorer.py at master · hand10ryo/ProgrammingBasicScorer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import os
import sys
import glob
import subprocess
import importlib

def convert(user_out):
    "初期設定のuserの出力を変換する関数"
    return user_out.decode("utf-8")

def load_data(path, filename):
    "初期設定のテストデータを読み込む関数"
    data_path = path + "/" + filename + "_test.txt"
    with open(data_path, "r") as f:
        data = f.read()
    return data

class Scorer:
    "スコア付けをするクラス"
    def __init__(self, path):
        self.path = path
        self.test_dir = path + "/submit"
        self.score_dict = {}

    def test_stdout(self, filename, convert=convert, testdata = None, max_score=1, stdin_file=None):
        """"
        課題が標準出力のときに使う関数

        - input
        filename : 【文字列】〜.pyの~を文字列で
        convert：【関数】上のconvert関数が初期設定。ユーザーの出力を弄りたければこれを設定
        testdata：【関数】 初期設定では上のload_dataでtestdataを読み込むが、文字列以外のデータを使いたければこれを設定する。
        max_score：【float】最大点

        - output
        None

        """
        test_file_list = glob.glob(self.test_dir+ "/*" + filename + ".py")
        for test_file in test_file_list:
            ID = test_file.split("/")[-1][:8]
            try:
                if stdin_file is None:
                    proc = subprocess.Popen(
                        ['python', test_file],
                        stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE
                    )
                    out, err = proc.communicate()
                else:
                    proc1 = subprocess.Popen(["python",stdin_file],stdout=subprocess.PIPE)
                    proc2 = subprocess.Popen(
                        ["python", test_file],
                        stdin = proc1.stdout,
                        stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE
                    )
                    proc1.stdout.close()
                    out, err = proc2.communicate()
                out_conv = convert(out)
                err = err.decode("utf-8")
            except:
                out = ""
                out_conv = ""
                err = "maybe timeout"

            if testdata is None:
                testdata = load_data(self.path, filename)

            if out_conv == testdata:
                print(ID, "OK")
                score = 1
            else:
                print("###########  code  ###############")
                with open(test_file, "r") as f:
                    print(f.read())
                print("###########  output ###############")
                print(out_conv)
                print("###########  error ###############")
                print(err)
                print()
                print(f"{ID}'s code is something wrong. Input partial score in [0,1]")
                score = float(input())

            if ID in self.score_dict.keys():
                self.score_dict[ID] += score * max_score
            else:
                self.score_dict[ID] = score * max_score

    def test_function(self, filename, funcname, convert=lambda x:x,
                      testdata_in = [], testdata_out=None, max_score=1):
        """"
        課題が関数のときに使う関数

        - input
        filename : 【文字列】〜.pyの~を文字列で
        funcname：【文字列課題の対象となる関数の名前
        convert：【関数】恒等写像が初期設定。ユーザーの出力を弄りたければこれを設定
        testdata_in：【リスト】対象の関数に入力する引数をリストで
        testdata_out：【オブジェクト】対象の関数の返り値をconvert関数に入力したときの想定する出力をオブジェクトで。
        　　　　　　　　　ただし"=="で正誤判定することに注意
        max_score：【float】最大点

        - output
        None

        """

        test_file_list = glob.glob(self.test_dir+ "/*" + filename + ".py")
        sys.path.append(self.test_dir)
        for test_file in test_file_list:
            pyfile = test_file.split("/")[-1][:-3]
            ID = pyfile[:8]

            try:
                mod = importlib.import_module(pyfile, package=None)
                func = getattr(mod, funcname)
                user_out = func(*testdata_in)
                out = convert(user_out)
            except:
                out = None

            if out == testdata_out:
                print(ID, "OK")
                score = 1
            else:
                print("###########  code  ###############")
                with open(test_file, "r") as f:
                    print(f.read())
                print("###########  output ###############")
                print(out)
                #print("###########  error ###############")
                #print(err)
                print()
                print(f"{ID}'s code is something wrong. Input partial score in [0,1]")
                score = float(input())

            if ID in self.score_dict.keys():
                self.score_dict[ID] += score * max_score
            else:
                self.score_dict[ID] = score * max_score