Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

添加SpiderOptions属性OneRequestDoneFirst，支持优先处理 request -> sub request ->… #244

Closed
wants to merge 1 commit into from
Closed

添加SpiderOptions属性OneRequestDoneFirst，支持优先处理 request -> sub request ->… #244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
添加SpiderOptions属性OneRequestDoneFirst，支持优先处理 request -> sub request ->… 
… sub request -> .... -> IDataFlow
  • Loading branch information
@capadong
capadong committed Mar 9, 2023
commit fbae3431fe3e14d01a2d7608e4c4d151cf02da3e
31 changes: 27 additions & 4 deletions src/DotnetSpider/Scheduler/QueueBfsScheduler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using DotnetSpider.Http;
using DotnetSpider.Infrastructure;
using DotnetSpider.Scheduler.Component;
using Microsoft.Extensions.Options;

namespace DotnetSpider.Scheduler
{
Expand All @@ -12,14 +13,17 @@ namespace DotnetSpider.Scheduler
/// </summary>
public class QueueBfsScheduler : SchedulerBase
{
readonly SpiderOptions _options;

private readonly List<Request> _requests =
new();

/// <summary>
/// 构造方法
/// </summary>
public QueueBfsScheduler(IRequestHasher requestHasher) : base(new FakeDuplicateRemover(), requestHasher)
public QueueBfsScheduler(IRequestHasher requestHasher, IOptions<SpiderOptions> options) : base(new FakeDuplicateRemover(), requestHasher)
{
_options = options.Value;
}

public override void Dispose()
Expand Down Expand Up @@ -50,10 +54,29 @@ protected override Task PushWhenNoDuplicate(Request request)
/// <returns>请求</returns>
protected override Task<IEnumerable<Request>> ImplDequeueAsync(int count = 1)
{
var requests = _requests.Take(count).ToArray();
if (requests.Length > 0)
Request[] requests = null;

if (_options.OneRequestDoneFirst)
{
requests = _requests
.OrderByDescending(x => x.Depth)
.Take(count).ToArray();

if (requests.Length > 0)
{
for (int i = 0; i < requests.Length; i++)
{
_requests.Remove(requests[i]);
}
}
}
else
{
_requests.RemoveRange(0, count);
requests = _requests.Take(count).ToArray();
if (requests.Length > 0)
{
_requests.RemoveRange(0, count);
}
}

return Task.FromResult(requests.Select(x => x.Clone()));
Expand Down
30 changes: 26 additions & 4 deletions src/DotnetSpider/Scheduler/QueueDistinctBfsScheduler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using DotnetSpider.Http;
using DotnetSpider.Infrastructure;
using DotnetSpider.Scheduler.Component;
using Microsoft.Extensions.Options;

namespace DotnetSpider.Scheduler
{
Expand All @@ -12,12 +13,14 @@ namespace DotnetSpider.Scheduler
/// </summary>
public class QueueDistinctBfsScheduler : SchedulerBase
{
readonly SpiderOptions _options;
private readonly List<Request> _requests =
new();

public QueueDistinctBfsScheduler(IDuplicateRemover duplicateRemover, IRequestHasher requestHasher)
public QueueDistinctBfsScheduler(IDuplicateRemover duplicateRemover, IRequestHasher requestHasher, IOptions<SpiderOptions> options)
: base(duplicateRemover, requestHasher)
{
_options = options.Value;
}

public override void Dispose()
Expand Down Expand Up @@ -47,10 +50,29 @@ protected override Task PushWhenNoDuplicate(Request request)
/// <returns>请求</returns>
protected override Task<IEnumerable<Request>> ImplDequeueAsync(int count = 1)
{
var requests = _requests.Take(count).ToArray();
if (requests.Length > 0)
Request[] requests = null;

if (_options.OneRequestDoneFirst)
{
requests = _requests
.OrderByDescending(x => x.Depth)
.Take(count).ToArray();

if (requests.Length > 0)
{
for (int i = 0; i < requests.Length; i++)
{
_requests.Remove(requests[i]);
}
}
}
else
{
_requests.RemoveRange(0, requests.Length);
requests = _requests.Take(count).ToArray();
if (requests.Length > 0)
{
_requests.RemoveRange(0, count);
}
}

return Task.FromResult(requests.Select(x => x.Clone()));
Expand Down
5 changes: 5 additions & 0 deletions src/DotnetSpider/SpiderOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,10 @@ public class SpiderOptions
/// 获取新代码的时间间隔
/// </summary>
public int RefreshProxy { get; set; } = 30;

/// <summary>
/// 优先处理 request -> sub request -> sub request -> .... -> IDataFlow
/// </summary>
public bool OneRequestDoneFirst { get; set; } = false;
}
}