axel: cookies, filename and referer support

[emiraga]

Hello, patch for axel cookies, output filename and referer support.

Code: Select all

diff -r -u t.org//content/flashgot/DMS.js t//content/flashgot/DMS.js
--- t.org//content/flashgot/DMS.js	2010-12-03 00:02:40.000000000 +0800
+++ t//content/flashgot/DMS.js	2010-12-19 13:37:55.391281001 +0800
@@ -2109,8 +2109,9 @@
     dm.askPath = ASK_NEVER;
   }
   
-  dm = new FlashGotDMX("Axel", "axel", '-n 4 [URL]');
+  dm = new FlashGotDMX("Axel", "axel", '-n 4 [URL] [-o FNAME] [-H Cookie:COOKIE] [-H Referer:REFERER]');
   dm.terminal = true;
+  dm.cookieSupport = true;
   dm.createJob = function(links, opType) {
     this.argsTemplate = this.argsTemplate.replace(/^-n \d+/, "-n " + this.getPref("connections", 4));  
     return this.__proto__.createJob.call(this, links , opType);

Please consider including this in your release.
Thanks.

[Giorgio Maone]

Many thanks, it will be included in 1.2.8.

[Giorgio Maone]

Little issue here: which version started supporting the -H argument?
Newest available on Ubuntu 8.04 LTS, for instance, is 1.0b which does NOT support it.

[emiraga]

http://alioth.debian.org/frs/?group_id=100070
I am surprised that Ubuntu would keep such old version. Ubuntu 10.10 has axel 2.4 as latest version.

Header support was added in version 2.0, finished in Sep 12 2008.
http://svn.debian.org/wsvn/axel/trunk/CHANGES

Can flashgot detect versions of downloaders?

[Giorgio Maone]

Can flashgot detect versions of downloaders?

Almost everything can be done with a bit of work
Please check latest development build, thanks.

[emiraga]

Almost everything can be done with a bit of work

Done, needs some testing.

What is weird is that axel 2.0 will identify itself as 1.1, but that should not be a problem since many people have 2.4.
Also it is weird that some ^M (carriage returns) appear in the patch, hopefully that won't give you problems (I can try uploading somewhere else).

Code: Select all

diff -r -u t.org//content/flashgot/DMS.js t//content/flashgot/DMS.js
--- t.org//content/flashgot/DMS.js	2010-12-16 15:49:54.000000000 +0800
+++ t//content/flashgot/DMS.js	2010-12-21 14:05:34.023420010 +0800
@@ -212,6 +212,7 @@
       }
       FlashGotDM.dmtests[this.exeName] = dmtest;
     } else dmtest = FlashGotDM.dmtests[this.exeName];
+    if(this.customVersionCheck) this.customVersionCheck(dmtest);
     return this._supported = dmtest.indexOf(this.name + "|OK") > -1;
   }
 ,
@@ -639,7 +640,7 @@
 
 
 // *** Unix-like DMS ***********************************************************
-function FlashGotDMX(name, cmd, argsTemplate) {
+function FlashGotDMX(name, cmd, argsTemplate, versionCmd) {
   if (arguments.length != 0) {
     this._init(name);
     const cmds = FlashGotDMX.prototype.unixCmds;
@@ -647,6 +648,7 @@
     this.unixCmd = cmd;
     if (argsTemplate) this.argsTemplate = argsTemplate;
     this.cookieSupport =  /\[.*?(?:CFILE|COOKIE).*?\]/.test(this.argsTemplate);
+    if(versionCmd) FlashGotDMX.prototype.versionCmds[name] = versionCmd;
   }
   if (fg.isMac) {
     this.createJobFile = FlashGotDMMac.prototype.createJobFile;
@@ -658,6 +660,7 @@
 FlashGotDMX.prototype.terminal = false;
 FlashGotDMX.prototype.askPath = [true, true, true];
 FlashGotDMX.prototype.unixCmds = {};
+FlashGotDMX.prototype.versionCmds = {};
 FlashGotDMX.prototype.__defineGetter__("unixShell", function() {
   var f = CC["@mozilla.org/file/local;1"].createInstance(CI.nsILocalFile);
   try {
@@ -677,12 +680,16 @@
 FlashGotDMX.prototype.argsTemplate = "[URL]";
 FlashGotDMX.prototype.launchSupportTest = function(testFile) {
   const cmds = this.unixCmds;
+  const vcmds = this.versionCmds;
   var script="(\n";
 
   for (var name in cmds) {
     cmd = cmds[name];
     script += " which \"" + cmd + "\" && echo '"
       + name + "|OK' || echo '" + name+"|KO'\n"; 
+    if(vcmds[name]) {
+      script += " which \"" + cmd + "\" && " + vcmds[name] + "\n";
+    }
   }
   script += ") > '" + testFile.path + "' 2>/dev/null\n"; 
   this.performJob(script, true);
@@ -2109,10 +2116,21 @@
     dm.askPath = ASK_NEVER;
   }
   
-  dm = new FlashGotDMX("Axel", "axel", '-n 4 [URL]');
+  dm = new FlashGotDMX("Axel", "axel", '-n 4 [URL] [-o FNAME]', 'axel --version');
   dm.terminal = true;
+  dm.version = 1;
+  dm.cookieSupport = true; //must be specified, even if not supported
+  dm.customVersionCheck = function(dmtest) {
+    var versionMsg = "Axel version ";
+    var start = dmtest.indexOf(versionMsg);
+    if(start < 0) return;
+    this.version = parseFloat(dmtest.substring(start+versionMsg.length));
+  }
   dm.createJob = function(links, opType) {
     this.argsTemplate = this.argsTemplate.replace(/^-n \d+/, "-n " + this.getPref("connections", 4));  
+    if(this.version >= 2) {
+      this.argsTemplate += " [-H Cookie:COOKIE] [-H Referer:REFERER]";
+    }
     return this.__proto__.createJob.call(this, links , opType);
   };
   

Comments welcome.

[Giorgio Maone]

Can flashgot detect versions of downloaders?

Almost everything can be done with a bit of work
Please check latest development build, thanks.

Did you actually check it?
It already had lazy (for performance reasons) version checking on the fly. I asked you to test it with Axel 2, since I already tested it on Axel 1.
Furthermore, if Axel 2.0 misidentify itself, I could change the check to grep for " -H " in axel --help, rather than reading the version number.

[emiraga]

Did you actually check it?
It already had lazy (for performance reasons) version checking on the fly.

I could not find anything like that in the code, and I did not find it mentioned in changelog. (I am working with version flashgot-1.2.7rc3.)

I asked you to test it with Axel 2, since I already tested it on Axel 1.

You weren't clear with that request (sorry to say that).

Furthermore, if Axel 2.0 misidentify itself, I could change the check to grep for " -H " in axel --help, rather than reading the version number.

Please point to me where is this version check happening in existing code, since I could not find it. I was thinking about feature detection as well, good idea after your reply I will figure out what to do.

[Giorgio Maone]

Oops, looks like I forgot to update the getit page.
Fixed, thanks.

[emiraga]

It works great from what I can see. And your fix is much simpler. Thanks for your effort and great product.

I am perfectly happy with current version of your code.
For the sake of completion, if you like feature detection.

Code: Select all

diff -r -u t.org//content/flashgot/DMS.js t//content/flashgot/DMS.js
--- t.org//content/flashgot/DMS.js	2010-12-20 11:59:56.000000000 +0800
+++ t//content/flashgot/DMS.js	2010-12-22 01:41:14.630743212 +0800
@@ -2114,22 +2114,20 @@
   dm.terminal = true;
   dm.createJob = function(links, opType) {
     this.argsTemplate = this.argsTemplate.replace(/\b-n \d+/, "-n " + this.getPref("connections", 4));
-    this._checkAxelVersion();
+    this._checkAxelFeatures();
     return this.__proto__.createJob.call(this, links , opType);
   };
-  dm._checkAxelVersion = function() {
+  dm._checkAxelFeatures = function() {
     const outFile = fg.tmpDir.clone();
-    outFile.append("axelVersion.txt");
-    this.performJob(this.unixCmd + " -V >'" + outFile.path + "' 2>&1", true);
-    var m = IO.readFile(outFile).match(/ (\d+\.\d)/);
-    this._axelVersion = m && parseFloat(m[1]) || 0;
-    if (this._axelVersion < 2) {
+    outFile.append("axelHelp.txt");
+    this.performJob(this.unixCmd + " -h >'" + outFile.path + "' 2>&1", true);
+    if(IO.readFile(outFile).indexOf("--header") < 0) {
       this.cookieSupport = false;
     } else {
       this.cookieSupport = true;
       this.argsTemplate = '[-H Cookie:COOKIE] [-H Referer:REFERER] ' + this.argsTemplate;
     }
-    this._checkAxelVersion = function() {};
+    this._checkAxelFeatures = function() {};
   }
   
  

[Giorgio Maone]

For the sake of completion, if you like feature detection.

I do prefer it if, as you said, 2.0 misreports its version number.
Thank you.

[emiraga]

Looking at file content/flashgot/DMS.js, class FlashGotDM.prototype and method download arround line 318.

this.checkCookieSupport is firstly called followed by this.performDownload which will call this.createJob. And createJob is overridden for axel to perform check for cookie support and set cookieSupport=true. Therefore, checkCookieSupport needs to be called again sometimes after feature detection.

I guess easiest fix is to do this:

Code: Select all

       this.cookieSupport = true;
       this.argsTemplate = '[-H Cookie:COOKIE] [-H Referer:REFERER] ' + this.argsTemplate;
+      this.checkCookieSupport();
     }

If you want to avoid double call to checkCookieSupport, some refactoring would be required.

P.S. I have no idea why was this working before, I am sorry for late report.

[Giorgio Maone]

If you want to avoid double call to checkCookieSupport, some refactoring would be required.

I don't mind the extra call, it's a microsecond thing and happens just once per session.

P.S. I have no idea why was this working before, I am sorry for late report.

Probably because it could break only the first download in a session, from then on everything works as expected.
Thank you for spotting it anyway.