Difference between revisions of "User:Kr/Concurrent Checkout"
From Apache OpenOffice Wiki
< User:Kr
m |
m (→The Code: Some code hightlightning ... though "awk" was not available as a language ;-)) |
||
(7 intermediate revisions by 2 users not shown) | |||
Line 1: | Line 1: | ||
This script has been pointed to by an [http://blogs.sun.com/GullFOSS/ GullFOSS] [http://blogs.sun.com/GullFOSS/entry/aawax_or_how_to_check entry]. | This script has been pointed to by an [http://blogs.sun.com/GullFOSS/ GullFOSS] [http://blogs.sun.com/GullFOSS/entry/aawax_or_how_to_check entry]. | ||
− | + | ==Parameters== | |
− | + | * CWS - the name of a CWS | |
− | + | * Milestone - the name of a milestone, this is mutual exclusive wrt CWS | |
− | + | * Master - the master | |
+ | * CVS - the CVS server, defaults to "anoncvs@anoncvs.services.openoffice.org" | ||
+ | * modules - the modules, defaults to "smoketestoo_native" | ||
+ | ==Examples== | ||
<pre> | <pre> | ||
− | # | + | # Check out a particular CWS |
+ | conco.awk -v Master=SRC680 -v CWS=bunoexttm | ||
− | + | # Check out a particular MWS | |
− | + | conco.awk -v Master=SRC680 -v Milestone=m215 | |
− | + | # More complex CWS check out | |
− | + | conco.awk -v modules=smoketestoo_native -v CVS="anoncvs@anoncvs.services.openoffice.org" -v Master=SRC680 -v CWS=bunoexttm | |
− | + | ||
+ | # More complex master check out | ||
+ | conco.awk -v modules=smoketestoo_native -v CVS="anoncvs@anoncvs.services.openoffice.org" -v Master=SRC680 -v Milestone=m203 | ||
+ | </pre> | ||
+ | |||
+ | ==Comments== | ||
+ | The handling of concurrent check outs is far from optimal, there is much room for improvement :-) | ||
+ | |||
+ | ==The Code== | ||
+ | <source lang="bash"> | ||
+ | #!/usr/bin/awk -f | ||
− | cvscmd = | + | function coModules(modules,dirname,tag,module,amodules,i,j) { |
+ | cvscmd = "cvs '-d:pserver:%s:/cvs' co %s %s > /dev/null &\n" | ||
split(modules,amodules) | split(modules,amodules) | ||
j = 0 # Maximum number of concurrent jobs. | j = 0 # Maximum number of concurrent jobs. | ||
for (i in amodules) { | for (i in amodules) { | ||
− | if (j | + | if (j <= 0) { |
− | j = | + | j = jobs # Maximum number of concurrent jobs. |
− | printf("wait\n") | | + | printf("wait\n") | Shell |
} | } | ||
− | printf(cvscmd,CVS,tag, | + | |
+ | module = amodules[i] | ||
+ | |||
+ | if (split(module,temp,"@") == 2) { | ||
+ | dirname = temp[1] | ||
+ | tag = "-r"temp[2] | ||
+ | } | ||
+ | else | ||
+ | dirname = module | ||
+ | |||
+ | printf(cvscmd,CVS,tag,dirname) | Shell | ||
j -- | j -- | ||
} | } | ||
− | printf("wait\n") | | + | printf("wait\n") | Shell |
− | close( | + | close(Shell) |
} | } | ||
− | + | function queryMilestone(cws,milestone) { | |
− | print " | + | print "export SOLARENV=./solenv" |& Shell |
− | print " | + | print "$SOLARENV/bin/cwsquery -m " Master " -c " cws " current" |& Shell |
− | + | Shell |& getline milestone | |
− | + | close(Shell) | |
− | + | return milestone | |
− | + | } | |
− | + | function queryModules(modules) { | |
− | + | print "export WORK_STAMP=" Master |& Shell | |
− | + | print "export CWS_WORK_STAMP=" CWS |& Shell | |
− | + | print "export SOLARENV=./solenv" |& Shell | |
− | + | print "echo `$SOLARENV/bin/cwsquery modules`" |& Shell | |
− | + | Shell |& getline modules | |
− | + | close(Shell) | |
− | + | ||
− | + | ||
− | + | return modules | |
− | + | ||
} | } | ||
− | function walkDependencies(modules,cnt,recs,line,nf,i,j,module) { | + | |
+ | function walkDependencies(modules,tags,smodules,amodules,cnt,recs,line,nf,i,j,module) { | ||
if (length(modules) <= 0) | if (length(modules) <= 0) | ||
return | return | ||
Line 63: | Line 86: | ||
# CO "build.lst" of wanted modules. | # CO "build.lst" of wanted modules. | ||
− | + | modules="" | |
for (i in amodules) { | for (i in amodules) { | ||
− | + | module = amodules[i] | |
− | smodules[ | + | |
− | + | if (!(module in smodules) || smodules[module] == 0) { | |
+ | smodules[module] = 1 # mark this module as tested | ||
+ | |||
+ | if (!(module in tags)) # no tagged yet? | ||
+ | tags[module] = Master Milestone | ||
+ | |||
+ | modules=modules " " module"/prj/build.lst@" tags[module] | ||
} | } | ||
} | } | ||
− | print "getting build.lst for " | + | print "getting build.lst for " modules " ..." |
− | + | coModules(modules) | |
+ | |||
modules="" | modules="" | ||
Line 79: | Line 109: | ||
# Check if the module is there now, otherwise ignore it. | # Check if the module is there now, otherwise ignore it. | ||
if (getline < (amodules[i]"/prj/build.lst") >= 0) { | if (getline < (amodules[i]"/prj/build.lst") >= 0) { | ||
− | |||
close(amodules[i]"/prj/build.lst") | close(amodules[i]"/prj/build.lst") | ||
+ | |||
+ | smodules[amodules[i]] = 2 # Mark this module as valid. | ||
# Get dependencies. | # Get dependencies. | ||
Line 93: | Line 124: | ||
j = 4 | j = 4 | ||
while(j < nf) { | while(j < nf) { | ||
− | + | if (split(recs[j],temp,":") == 2) | |
− | + | module = temp[2] | |
− | + | else | |
− | else | + | module = temp[1] |
− | + | ||
− | |||
if (!(module in smodules)) { | if (!(module in smodules)) { | ||
modules = modules " " module | modules = modules " " module | ||
Line 113: | Line 142: | ||
} | } | ||
− | walkDependencies(modules) | + | walkDependencies(modules,tags,smodules) |
} | } | ||
− | </ | + | |
+ | function findDependencies(modules,tags,smodules,temp) { | ||
+ | walkDependencies(modules,tags,smodules) | ||
+ | |||
+ | modules="" | ||
+ | asorti(smodules,temp) | ||
+ | for (i in temp) { | ||
+ | if (smodules[temp[i]] == 2) { | ||
+ | modules=modules " " temp[i] | ||
+ | |||
+ | if (temp[i] in tags) # need to tag | ||
+ | modules = modules "@" tags[temp[i]] | ||
+ | else | ||
+ | modules = modules "@" Master Milestone | ||
+ | } | ||
+ | } | ||
+ | |||
+ | return modules | ||
+ | } | ||
+ | |||
+ | BEGIN { | ||
+ | Shell ="/bin/sh" | ||
+ | DefModules = "smoketestoo_native" | ||
+ | DefCVS = "anoncvs@anoncvs.services.openoffice.org" | ||
+ | |||
+ | # Fall back for modules. | ||
+ | if (modules == "") { | ||
+ | print "No modules given (-v modules=<name>*)" | ||
+ | print "falling back to " DefModules | ||
+ | print | ||
+ | |||
+ | modules = DefModules | ||
+ | } | ||
+ | |||
+ | # Fall back for CVS server. | ||
+ | if (CVS == "") { | ||
+ | print "No CVS server given (-v CVS=<server>)" | ||
+ | print "falling back to " DefCVS | ||
+ | print | ||
+ | |||
+ | CVS = DefCVS | ||
+ | } | ||
+ | |||
+ | if (jobs == "") { | ||
+ | if (CVS == "anoncvs@anoncvs.services.openoffice.org") | ||
+ | jobs = 3 | ||
+ | else | ||
+ | jobs = 35 | ||
+ | } | ||
+ | |||
+ | if (CWS != "") { | ||
+ | coModules("solenv@" Master) | ||
+ | Milestone = queryMilestone(CWS) | ||
+ | cwsModules = queryModules() | ||
+ | print "rm -r solenv" | Shell | ||
+ | close(Shell) | ||
+ | |||
+ | split(cwsModules,temp) | ||
+ | for (i in temp) { | ||
+ | tags[temp[i]] = "cws_" tolower(Master) "_" tolower(CWS) | ||
+ | } | ||
+ | } | ||
+ | |||
+ | print "modules: " modules | ||
+ | print "CVS: " CVS | ||
+ | print "Master:" Master | ||
+ | print "cws:" CWS | ||
+ | print "jobs: " jobs | ||
+ | print "milestone:" Milestone | ||
+ | |||
+ | Milestone = "_" Milestone | ||
+ | |||
+ | |||
+ | # Unfortunately, there are some modules, which somehow are not reachable by | ||
+ | # the dependencies. | ||
+ | smodules["config_office" ]=2 | ||
+ | smodules["dmake" ]=2 | ||
+ | smodules["default_images" ]=2 | ||
+ | smodules["external_images"]=2 | ||
+ | smodules["lingucomponent" ]=2 | ||
+ | modules=findDependencies(modules,tags,smodules) | ||
+ | |||
+ | print "checking out needed modules: " modules " ..." | ||
+ | coModules(modules, Master) | ||
+ | } | ||
+ | </source> |
Latest revision as of 14:32, 3 December 2008
This script has been pointed to by an GullFOSS entry.
Contents
Parameters
- CWS - the name of a CWS
- Milestone - the name of a milestone, this is mutual exclusive wrt CWS
- Master - the master
- CVS - the CVS server, defaults to "anoncvs@anoncvs.services.openoffice.org"
- modules - the modules, defaults to "smoketestoo_native"
Examples
# Check out a particular CWS conco.awk -v Master=SRC680 -v CWS=bunoexttm # Check out a particular MWS conco.awk -v Master=SRC680 -v Milestone=m215 # More complex CWS check out conco.awk -v modules=smoketestoo_native -v CVS="anoncvs@anoncvs.services.openoffice.org" -v Master=SRC680 -v CWS=bunoexttm # More complex master check out conco.awk -v modules=smoketestoo_native -v CVS="anoncvs@anoncvs.services.openoffice.org" -v Master=SRC680 -v Milestone=m203
Comments
The handling of concurrent check outs is far from optimal, there is much room for improvement :-)
The Code
#!/usr/bin/awk -f function coModules(modules,dirname,tag,module,amodules,i,j) { cvscmd = "cvs '-d:pserver:%s:/cvs' co %s %s > /dev/null &\n" split(modules,amodules) j = 0 # Maximum number of concurrent jobs. for (i in amodules) { if (j <= 0) { j = jobs # Maximum number of concurrent jobs. printf("wait\n") | Shell } module = amodules[i] if (split(module,temp,"@") == 2) { dirname = temp[1] tag = "-r"temp[2] } else dirname = module printf(cvscmd,CVS,tag,dirname) | Shell j -- } printf("wait\n") | Shell close(Shell) } function queryMilestone(cws,milestone) { print "export SOLARENV=./solenv" |& Shell print "$SOLARENV/bin/cwsquery -m " Master " -c " cws " current" |& Shell Shell |& getline milestone close(Shell) return milestone } function queryModules(modules) { print "export WORK_STAMP=" Master |& Shell print "export CWS_WORK_STAMP=" CWS |& Shell print "export SOLARENV=./solenv" |& Shell print "echo `$SOLARENV/bin/cwsquery modules`" |& Shell Shell |& getline modules close(Shell) return modules } function walkDependencies(modules,tags,smodules,amodules,cnt,recs,line,nf,i,j,module) { if (length(modules) <= 0) return split(modules,amodules) # CO "build.lst" of wanted modules. modules="" for (i in amodules) { module = amodules[i] if (!(module in smodules) || smodules[module] == 0) { smodules[module] = 1 # mark this module as tested if (!(module in tags)) # no tagged yet? tags[module] = Master Milestone modules=modules " " module"/prj/build.lst@" tags[module] } } print "getting build.lst for " modules " ..." coModules(modules) modules="" # Walk the dependencies. for (i in amodules) { # Check if the module is there now, otherwise ignore it. if (getline < (amodules[i]"/prj/build.lst") >= 0) { close(amodules[i]"/prj/build.lst") smodules[amodules[i]] = 2 # Mark this module as valid. # Get dependencies. do { cnt = getline line < (amodules[i]"/prj/build.lst") } while(cnt >= 0 && line ~ /^#.*/) close(amodules[i]"/prj/build.lst") if (cnt >= 0) { nf = split(line, recs) j = 4 while(j < nf) { if (split(recs[j],temp,":") == 2) module = temp[2] else module = temp[1] if (!(module in smodules)) { modules = modules " " module smodules[module] = 0 } j=j + 1 } } } else smodules[amodules[i]] = 1 # Mark this module as invalid. } walkDependencies(modules,tags,smodules) } function findDependencies(modules,tags,smodules,temp) { walkDependencies(modules,tags,smodules) modules="" asorti(smodules,temp) for (i in temp) { if (smodules[temp[i]] == 2) { modules=modules " " temp[i] if (temp[i] in tags) # need to tag modules = modules "@" tags[temp[i]] else modules = modules "@" Master Milestone } } return modules } BEGIN { Shell ="/bin/sh" DefModules = "smoketestoo_native" DefCVS = "anoncvs@anoncvs.services.openoffice.org" # Fall back for modules. if (modules == "") { print "No modules given (-v modules=<name>*)" print "falling back to " DefModules print modules = DefModules } # Fall back for CVS server. if (CVS == "") { print "No CVS server given (-v CVS=<server>)" print "falling back to " DefCVS print CVS = DefCVS } if (jobs == "") { if (CVS == "anoncvs@anoncvs.services.openoffice.org") jobs = 3 else jobs = 35 } if (CWS != "") { coModules("solenv@" Master) Milestone = queryMilestone(CWS) cwsModules = queryModules() print "rm -r solenv" | Shell close(Shell) split(cwsModules,temp) for (i in temp) { tags[temp[i]] = "cws_" tolower(Master) "_" tolower(CWS) } } print "modules: " modules print "CVS: " CVS print "Master:" Master print "cws:" CWS print "jobs: " jobs print "milestone:" Milestone Milestone = "_" Milestone # Unfortunately, there are some modules, which somehow are not reachable by # the dependencies. smodules["config_office" ]=2 smodules["dmake" ]=2 smodules["default_images" ]=2 smodules["external_images"]=2 smodules["lingucomponent" ]=2 modules=findDependencies(modules,tags,smodules) print "checking out needed modules: " modules " ..." coModules(modules, Master) }