Skip to content

Commit

Permalink
Finishing of grouped test
Browse files Browse the repository at this point in the history
- Tests are grouped now
- Also shows in the files ones where there was a non 0 return code of
CCX.
  • Loading branch information
wforums committed Jun 8, 2015
1 parent e3df59e commit 1d60bb9
Show file tree
Hide file tree
Showing 8 changed files with 229 additions and 94 deletions.
104 changes: 85 additions & 19 deletions CCExtractorTester/Analyzers/Tester.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,22 @@ namespace CCExtractorTester
/// </summary>
public class Tester
{
/// <summary>
/// The name of the loaded file.
/// </summary>
/// <value>The name of the loaded file.</value>
public String LoadedFileName { get; private set; }
/// <summary>
/// A list of the TestEntry instances that will be processed.
/// </summary>
/// <value>The entries.</value>
public List<TestEntry> Entries { get; private set; }
/// <summary>
/// Contains the multi test list.
/// </summary>
/// <value>The multi test.</value>
public List<string> MultiTest { get; private set; }
/// <summary>
/// Gets or sets the progress reporter that will be used.
/// </summary>
/// <value>The progress reporter.</value>
Expand Down Expand Up @@ -52,6 +62,7 @@ public class Tester
/// <param name="logger">The logger that will be used.</param>
public Tester(ConfigurationSettings cfg,ILogger logger){
Entries = new List<TestEntry> ();
MultiTest = new List<String> ();
ProgressReporter = NullProgressReporter.Instance;
Config = cfg;
Logger = logger;
Expand Down Expand Up @@ -146,7 +157,9 @@ void loadAndParseXML (string xmlFileName)
using(FileStream fs = new FileStream(xmlFileName,FileMode.Open)){
doc.Load (fs);
XmlNodeList testNodes = doc.SelectNodes ("//test");
FileInfo fi = new FileInfo (xmlFileName);
if (testNodes.Count > 0) {
LoadedFileName = fi.Name.Replace(".xml","");
foreach (XmlNode node in testNodes) {
XmlNode sampleFile = node.SelectSingleNode ("sample");
XmlNode command = node.SelectSingleNode ("cmd");
Expand All @@ -157,7 +170,8 @@ void loadAndParseXML (string xmlFileName)
// Dealing with multi file
foreach (XmlNode node in doc.SelectNodes ("//testfile")) {
String testFileLocation = ConvertFolderDelimiters(node.SelectSingleNode ("location").InnerText);
// TODO: add to separate list, which then will be parsed following the same pattern.
testFileLocation = Path.Combine (fi.DirectoryName, testFileLocation);
MultiTest.Add (testFileLocation);
}
}
}
Expand All @@ -176,7 +190,7 @@ void ValidateXML (string xmlFileName)
ValidateAgainstSchema (xmlFileName, Resources.tests);
} catch(XmlSchemaValidationException){
try {
ValidateAgainstSchema (xmlFileName, Resources.multitest);
ValidateAgainstSchema (xmlFileName, Resources.multitest);
} catch(XmlSchemaValidationException){
throw new InvalidDataException ("Given XML is neither a test XML file nor a multitest XML file.");
}
Expand All @@ -189,7 +203,7 @@ void ValidateXML (string xmlFileName)
/// <param name="xmlFileName">Xml file name.</param>
/// <param name="xmlSchema">Xml schema.</param>
private void ValidateAgainstSchema(string xmlFileName, string xmlSchema){
using (StringReader sr = new StringReader (Resources.tests)) {
using (StringReader sr = new StringReader (xmlSchema)) {
XmlReader r = XmlReader.Create (sr);
XmlReaderSettings settings = new XmlReaderSettings ();
settings.Schemas.Add (null, r);
Expand Down Expand Up @@ -237,8 +251,6 @@ public void RunTests(){
throw new InvalidOperationException ("Sample folder does not exist!");
}

LoadComparer ();

String location = System.Reflection.Assembly.GetExecutingAssembly ().Location;
location = location.Remove (location.LastIndexOf (Path.DirectorySeparatorChar));
String temporaryFolder = Config.GetAppSetting ("temporaryFolder");
Expand All @@ -252,6 +264,51 @@ public void RunTests(){
Logger.Info ("Using threading");
}

if (MultiTest.Count > 0) {
// Override ReportFolder and create subdirectory for it if necessary
String subFolder = Path.Combine(Config.GetAppSetting("ReportFolder"),"Testsuite_Report_"+DateTime.Now.ToString("yyyy-MM-dd_HHmmss"));
if (!Directory.Exists (subFolder)) {
Directory.CreateDirectory (subFolder);
}
Logger.Info ("Multitest, overriding report folder to: " + subFolder);
Config.SetAppSetting ("ReportFolder", subFolder);
// Run through test files
StringBuilder sb = new StringBuilder(@"
<html>
<head>
<title>Test suite result index</title>
<style>.green { background-color: green; } .red { background-color: red; }</style>
</head>
<body>
<table>
<tr>
<th>Report name</th>
<th>Tests passed</th>
</tr>");
foreach (string s in MultiTest) {
Entries.Clear();
loadAndParseXML (s);
int nrTests = Entries.Count;
Tuple<int,string> singleTest = RunSingleFileTests (useThreading, cce, location, sourceFolder);
sb.AppendFormat (
@"<tr><td><a href=""{0}"">{1}</a></td><td class=""{2}"">{3}/{4}</td></tr>",
singleTest.Item2,LoadedFileName,
(singleTest.Item1 == nrTests)?"green":"red",
singleTest.Item1,nrTests
);
}
sb.Append ("</table></body></html>");
using (StreamWriter sw = new StreamWriter (Path.Combine (subFolder, "index.html"))) {
sw.WriteLine (sb.ToString ());
}
} else {
RunSingleFileTests (useThreading, cce, location, sourceFolder);
}
}

Tuple<int,string> RunSingleFileTests(bool useThreading, String cce, String location, String sourceFolder){
LoadComparer ();

int i = 1;
SetUpTestEntryProcessing (cce, location,sourceFolder,Entries.Count);
ManualResetEvent[] mres = new ManualResetEvent[Entries.Count];
Expand All @@ -275,8 +332,17 @@ public void RunTests(){
}
DateTime end = DateTime.Now;
Logger.Info ("Runtime: "+(end.Subtract(start)).ToString());
Comparer.SaveReport (Config.GetAppSetting ("ReportFolder"), new ResultData (){ CCExtractorVersion = cce + " on " + DateTime.Now.ToShortDateString () });
String reportName = Comparer.SaveReport (
Config.GetAppSetting ("ReportFolder"),
new ResultData (){
FileName = LoadedFileName,
CCExtractorVersion = cce + " on " + DateTime.Now.ToShortDateString (),
StartTime = start
}
);
int unchanged = Comparer.GetSuccessNumber();
Comparer = null;
return Tuple.Create(unchanged,reportName);
}

/// <summary>
Expand Down Expand Up @@ -350,20 +416,20 @@ public void Process(object state){

RunData rd = runner.Run (command,processError,processOutput);

if (rd.ExitCode == 0) {
try {
comparer.CompareAndAddToResult (
new CompareData(){
ProducedFile = producedFile,
CorrectFile = expectedResultFile,
SampleFile = sampleFile,
Command = te.Command,
RunTime= rd.Runtime
});
} catch(Exception e){
logger.Error (e);
}
try {
comparer.CompareAndAddToResult (
new CompareData () {
ProducedFile = producedFile,
CorrectFile = expectedResultFile,
SampleFile = sampleFile,
Command = te.Command,
RunTime = rd.Runtime,
ExitCode = rd.ExitCode
});
} catch (Exception e) {
logger.Error (e);
}

progressReporter.showProgressMessage (String.Format ("Finished entry {0} with exit code: {1}", current,rd.ExitCode));
eventX.Set ();
}
Expand Down
2 changes: 1 addition & 1 deletion CCExtractorTester/CCExtractorTester.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<WarningLevel>4</WarningLevel>
<PlatformTarget>x86</PlatformTarget>
<ConsolePause>false</ConsolePause>
<Commandlineparameters>-t xsd.xml</Commandlineparameters>
<Commandlineparameters>-t "E:\CCExtractor repository\TestFiles\TestAll.xml"</Commandlineparameters>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
<DebugType>full</DebugType>
Expand Down
5 changes: 5 additions & 0 deletions CCExtractorTester/Comparers/CompareData.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,10 @@ public class CompareData
/// </summary>
/// <value>The run time.</value>
public TimeSpan RunTime { get; set; }
/// <summary>
/// Gets or sets the exit code.
/// </summary>
/// <value>The exit code.</value>
public int ExitCode { get; set; }
}
}
59 changes: 37 additions & 22 deletions CCExtractorTester/Comparers/DiffLinuxComparer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,49 +35,64 @@ public void CompareAndAddToResult (CompareData data)
Builder.AppendLine ("Time needed for this entry: "+data.RunTime.ToString());
Builder.AppendLine ("Used command: " + data.Command);
Builder.AppendLine ("Sample file: " + data.SampleFile);
if (!Hasher.filesAreEqual (data.CorrectFile, data.ProducedFile)) {
ProcessStartInfo psi = new ProcessStartInfo ("diff");
psi.UseShellExecute = false;
psi.RedirectStandardError = true;
psi.RedirectStandardOutput = true;
psi.CreateNoWindow = true;
if (data.ExitCode != 0) {
Builder.AppendLine ("CCExtractor quit with exit code " + data.ExitCode);
} else {
if (!Hasher.filesAreEqual (data.CorrectFile, data.ProducedFile)) {
ProcessStartInfo psi = new ProcessStartInfo ("diff");
psi.UseShellExecute = false;
psi.RedirectStandardError = true;
psi.RedirectStandardOutput = true;
psi.CreateNoWindow = true;

psi.Arguments = String.Format (@"-y ""{0}"" ""{1}""", data.CorrectFile, data.ProducedFile);
Process p = new Process ();
p.StartInfo = psi;
p.ErrorDataReceived += processError;
p.OutputDataReceived += processOutput;
p.Start ();
p.BeginOutputReadLine ();
p.BeginErrorReadLine ();
while (!p.HasExited) {
Thread.Sleep (1000);
psi.Arguments = String.Format (@"-y ""{0}"" ""{1}""", data.CorrectFile, data.ProducedFile);
Process p = new Process ();
p.StartInfo = psi;
p.ErrorDataReceived += processError;
p.OutputDataReceived += processOutput;
p.Start ();
p.BeginOutputReadLine ();
p.BeginErrorReadLine ();
while (!p.HasExited) {
Thread.Sleep (1000);
}
} else {
Builder.AppendLine ("Samples are equal (hash)");
}
} else {
Builder.AppendLine ("Samples are equal (hash)");
}
}
/// <summary>
/// Gets the name of the report file.
/// </summary>
/// <returns>The report file name.</returns>
public string GetReportFileName ()
/// <param name="data">Data.</param>
public string GetReportFileName (ResultData data)
{
return "Report_" + DateTime.Now.ToFileTime () + ".txt";
return "Report_" + data.FileName + "_" + data.StartTime.ToString("yyyy-MM-dd_HHmmss") + ".txt";
}
/// <summary>
/// Saves the report to a given file, with some extra data provided.
/// </summary>
/// <param name="pathToFolder">Path to folder to save the report in</param>
/// <param name="data">The extra result data that should be in the report.</param>
public void SaveReport (string pathToFolder, ResultData data)
public String SaveReport (string pathToFolder, ResultData data)
{
using (StreamWriter sw = new StreamWriter(Path.Combine(pathToFolder,GetReportFileName()))) {
String reportName = GetReportFileName (data);
using (StreamWriter sw = new StreamWriter(Path.Combine(pathToFolder,reportName))) {
sw.WriteLine ("Report generated for version " + data.CCExtractorVersion);
sw.WriteLine (Builder.ToString ());
}
return reportName;
}

/// <summary>
/// Gets the success number.
/// </summary>
/// <returns>The success number.</returns>
public int GetSuccessNumber ()
{
return -1; // Cannot implement this in this comparer.
}
#endregion
/// <summary>
/// Processes an error received by the diff command
Expand Down
Loading

0 comments on commit 1d60bb9

Please sign in to comment.