1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
import socket
import thread
try:
import cPickle as pickle
except ImportError:
import pickle
import StringIO
import os
import sys
import time
import portage
import random
import collagen.protocol as protocol
import db.main.models as dbm
from db import DjangoDB
class MatchboxServer(object):
"""
Class representing master Matchbox server deciding what needs to
be compiled. Tinderboxes connect to this server and ask for
packages to compile. When they return package contents (or errors)
Matchbox adds this information to database using DjangoDB backend.
"""
def __init__(self, host, port):
self.host = host
self.port = port
self.sock = None
self.db = DjangoDB()
self.portsettings = portage.config(clone=portage.settings)
def start_server(self):
"""
Starts matchbox server waiting for Tinderbox connections
"""
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
print "matchbox: Unable to bind socket"
self.sock = None
return None
self.sock.bind((self.host, self.port))
self.sock.listen(5)
while 1:
client_socket, address = self.sock.accept()
print "connection from: ", address
thread.start_new_thread(self.client_handler, (client_socket, address))
def stop_server(self):
if self.sock:
self.sock.close()
self.sock = None
def client_handler(self, client_socket, client_address):
"""
Service for one Tinderbox connection accepting
commands/replies
@param client_socket: socket of client connection
@type client_socket: socket
@param client_address: (address,port) tuple of client
@type client_address: tuple
"""
while 1:
buffer = client_socket.recv(4096)
data = ""
while len(buffer) == 4096:
data = data + buffer
buffer = client_socket.recv(4096)
data = data + buffer
if not data:
break;
# potentially dangerous if coming from malicious source
command = pickle.loads(data)
if type(command) is protocol.GetNextPackage:
print "returning next package to compile"
# TODO get real package from database with missing info
repl = protocol.GetNextPackageReply(self._get_next_package(), None, None)
print "name: %s" % repl.package_name
client_socket.sendall(pickle.dumps(repl))
elif type(command) is protocol.AddPackageInfo:
fout = open("/tmp/collagen_%s" % str(time.time()), "w")
for pi in command.package_infos:
print "adding package info"
print pi
sys.stdout = fout
print pi
sys.stdout = sys.__stdout__
self._db_add_package_info(pi, client_address)
fout.close()
# TODO
else:
print "unknown command: %s" % command
print "closing client connection"
client_socket.close()
def _get_next_package(self):
"""
Returns category/package string of next pacakge to be compiled
by tinderbox(en)
@returns: category/package string
@rtype: string
"""
override = self.__get_override_package()
if override:
return override
categories = os.listdir(self.portsettings["PORTDIR"])
cat_ind = random.randint(0,len(categories)-1)
selcat = categories[cat_ind]
checkdir = "%s/%s" % (self.portsettings["PORTDIR"], selcat)
if not os.path.isdir(checkdir):
return self._get_next_package()
packages = os.listdir(checkdir)
pkg_ind = random.randint(0,len(packages)-1)
selpkg = packages[pkg_ind]
checkdir = "%s/%s/%s" % (self.portsettings["PORTDIR"], selcat, selpkg)
if not os.path.isdir(checkdir):
return self._get_next_package()
return "%s/%s" % (selcat,selpkg)
def _db_add_package_info(self, pi, tinderbox_address):
db = self.db
pcat, pname = portage.catsplit(pi.name)
pid = db.add_package(pname)
package = dbm.Package.objects.get(pk=pid)
cid = db.add_category(pcat)
category = dbm.PackageCategory.objects.get(pk=cid)
pvid = db.add_package_version(package.id, category.id, pi.version)
packageversion = dbm.PackageVersion.objects.get(pk=pvid)
packageversion.dependencies.clear()
#we will update deps after all package infos have been inserted
profileid = db.add_portage_profile(pi.profile)
profile = dbm.PortageProfile.objects.get(pk=profileid)
tid = db.add_tinderbox(tinderbox_address[0])
tinderbox = dbm.Tinderbox.objects.get(pk=tid)
useflag_ids = []
if not pi.use_flags:
pi.use_flags = []
for useflag in pi.use_flags:
useflag_ids.append(db.add_useflag(useflag))
ecode = 0
if pi.error:
ecode = pi.error
ppid = db.add_packageproperties(pvid, profile.id, tinderbox.id, ecode)
db.add_useflags_to_packageproperies(ppid, useflag_ids)
for key in pi.attachments.keys():
db.add_attachment(ppid, key, pi.attachments[key], 'text/plain')
db.add_contents_to_packageproperties(ppid, pi.content)
def __get_override_package(self):
"""
Function to simplify debugging. If file /tmp/matchbox_override
exists it reads first line and returns it. It's used to force
selection of certain package as next package for tinderbox to compile
"""
try:
line = None
fin = open("/tmp/matchbox_override","r")
line = fin.readline()
line = line.strip()
except:
pass
finally:
return line
|
GitWeb