File: /usr/src/linux/drivers/scsi/scsi_merge.c
1 /*
2 * scsi_merge.c Copyright (C) 1999 Eric Youngdale
3 *
4 * SCSI queueing library.
5 * Initial versions: Eric Youngdale (eric@andante.org).
6 * Based upon conversations with large numbers
7 * of people at Linux Expo.
8 * Support for dynamic DMA mapping: Jakub Jelinek (jakub@redhat.com).
9 */
10
11 /*
12 * This file contains queue management functions that are used by SCSI.
13 * Typically this is used for several purposes. First, we need to ensure
14 * that commands do not grow so large that they cannot be handled all at
15 * once by a host adapter. The various flavors of merge functions included
16 * here serve this purpose.
17 *
18 * Note that it would be quite trivial to allow the low-level driver the
19 * flexibility to define it's own queue handling functions. For the time
20 * being, the hooks are not present. Right now we are just using the
21 * data in the host template as an indicator of how we should be handling
22 * queues, and we select routines that are optimized for that purpose.
23 *
24 * Some hosts do not impose any restrictions on the size of a request.
25 * In such cases none of the merge functions in this file are called,
26 * and we allow ll_rw_blk to merge requests in the default manner.
27 * This isn't guaranteed to be optimal, but it should be pretty darned
28 * good. If someone comes up with ideas of better ways of managing queues
29 * to improve on the default behavior, then certainly fit it into this
30 * scheme in whatever manner makes the most sense. Please note that
31 * since each device has it's own queue, we have considerable flexibility
32 * in queue management.
33 */
34
35 #define __NO_VERSION__
36 #include <linux/config.h>
37 #include <linux/module.h>
38
39 #include <linux/sched.h>
40 #include <linux/timer.h>
41 #include <linux/string.h>
42 #include <linux/slab.h>
43 #include <linux/ioport.h>
44 #include <linux/kernel.h>
45 #include <linux/stat.h>
46 #include <linux/blk.h>
47 #include <linux/interrupt.h>
48 #include <linux/delay.h>
49 #include <linux/smp_lock.h>
50
51
52 #define __KERNEL_SYSCALLS__
53
54 #include <linux/unistd.h>
55
56 #include <asm/system.h>
57 #include <asm/irq.h>
58 #include <asm/dma.h>
59 #include <asm/io.h>
60
61 #include "scsi.h"
62 #include "hosts.h"
63 #include "constants.h"
64 #include <scsi/scsi_ioctl.h>
65
66 /*
67 * This means that bounce buffers cannot be allocated in chunks > PAGE_SIZE.
68 * Ultimately we should get away from using a dedicated DMA bounce buffer
69 * pool, and we should instead try and use kmalloc() instead. If we can
70 * eliminate this pool, then this restriction would no longer be needed.
71 */
72 #define DMA_SEGMENT_SIZE_LIMITED
73
74 #ifdef CONFIG_SCSI_DEBUG_QUEUES
75 /*
76 * Enable a bunch of additional consistency checking. Turn this off
77 * if you are benchmarking.
78 */
79 static int dump_stats(struct request *req,
80 int use_clustering,
81 int dma_host,
82 int segments)
83 {
84 struct buffer_head *bh;
85
86 /*
87 * Dump the information that we have. We know we have an
88 * inconsistency.
89 */
90 printk("nr_segments is %x\n", req->nr_segments);
91 printk("counted segments is %x\n", segments);
92 printk("Flags %d %d\n", use_clustering, dma_host);
93 for (bh = req->bh; bh->b_reqnext != NULL; bh = bh->b_reqnext)
94 {
95 printk("Segment 0x%p, blocks %d, addr 0x%lx\n",
96 bh,
97 bh->b_size >> 9,
98 virt_to_phys(bh->b_data - 1));
99 }
100 panic("Ththththaats all folks. Too dangerous to continue.\n");
101 }
102
103
104 /*
105 * Simple sanity check that we will use for the first go around
106 * in order to ensure that we are doing the counting correctly.
107 * This can be removed for optimization.
108 */
109 #define SANITY_CHECK(req, _CLUSTER, _DMA) \
110 if( req->nr_segments != __count_segments(req, _CLUSTER, _DMA, NULL) ) \
111 { \
112 printk("Incorrect segment count at 0x%p", current_text_addr()); \
113 dump_stats(req, _CLUSTER, _DMA, __count_segments(req, _CLUSTER, _DMA, NULL)); \
114 }
115 #else
116 #define SANITY_CHECK(req, _CLUSTER, _DMA)
117 #endif
118
119 static void dma_exhausted(Scsi_Cmnd * SCpnt, int i)
120 {
121 int jj;
122 struct scatterlist *sgpnt;
123 int consumed = 0;
124
125 sgpnt = (struct scatterlist *) SCpnt->request_buffer;
126
127 /*
128 * Now print out a bunch of stats. First, start with the request
129 * size.
130 */
131 printk("dma_free_sectors:%d\n", scsi_dma_free_sectors);
132 printk("use_sg:%d\ti:%d\n", SCpnt->use_sg, i);
133 printk("request_bufflen:%d\n", SCpnt->request_bufflen);
134 /*
135 * Now dump the scatter-gather table, up to the point of failure.
136 */
137 for(jj=0; jj < SCpnt->use_sg; jj++)
138 {
139 printk("[%d]\tlen:%d\taddr:%p\talt:%p\n",
140 jj,
141 sgpnt[jj].length,
142 sgpnt[jj].address,
143 sgpnt[jj].alt_address);
144 if( sgpnt[jj].alt_address != NULL )
145 {
146 consumed = (sgpnt[jj].length >> 9);
147 }
148 }
149 printk("Total %d sectors consumed\n", consumed);
150 panic("DMA pool exhausted");
151 }
152
153 /*
154 * FIXME(eric) - the original disk code disabled clustering for MOD
155 * devices. I have no idea why we thought this was a good idea - my
156 * guess is that it was an attempt to limit the size of requests to MOD
157 * devices.
158 */
159 #define CLUSTERABLE_DEVICE(SH,SD) (SH->use_clustering && \
160 SD->type != TYPE_MOD)
161
162 /*
163 * This entire source file deals with the new queueing code.
164 */
165
166 /*
167 * Function: __count_segments()
168 *
169 * Purpose: Prototype for queue merge function.
170 *
171 * Arguments: q - Queue for which we are merging request.
172 * req - request into which we wish to merge.
173 * use_clustering - 1 if this host wishes to use clustering
174 * dma_host - 1 if this host has ISA DMA issues (bus doesn't
175 * expose all of the address lines, so that DMA cannot
176 * be done from an arbitrary address).
177 * remainder - used to track the residual size of the last
178 * segment. Comes in handy when we want to limit the
179 * size of bounce buffer segments to PAGE_SIZE.
180 *
181 * Returns: Count of the number of SG segments for the request.
182 *
183 * Lock status:
184 *
185 * Notes: This is only used for diagnostic purposes.
186 */
187 __inline static int __count_segments(struct request *req,
188 int use_clustering,
189 int dma_host,
190 int * remainder)
191 {
192 int ret = 1;
193 int reqsize = 0;
194 struct buffer_head *bh;
195 struct buffer_head *bhnext;
196
197 if( remainder != NULL ) {
198 reqsize = *remainder;
199 }
200
201 /*
202 * Add in the size increment for the first buffer.
203 */
204 bh = req->bh;
205 #ifdef DMA_SEGMENT_SIZE_LIMITED
206 if( reqsize + bh->b_size > PAGE_SIZE ) {
207 ret++;
208 reqsize = bh->b_size;
209 } else {
210 reqsize += bh->b_size;
211 }
212 #else
213 reqsize += bh->b_size;
214 #endif
215
216 for (bh = req->bh, bhnext = bh->b_reqnext;
217 bhnext != NULL;
218 bh = bhnext, bhnext = bh->b_reqnext) {
219 if (use_clustering) {
220 /*
221 * See if we can do this without creating another
222 * scatter-gather segment. In the event that this is a
223 * DMA capable host, make sure that a segment doesn't span
224 * the DMA threshold boundary.
225 */
226 if (dma_host &&
227 virt_to_phys(bhnext->b_data) - 1 == ISA_DMA_THRESHOLD) {
228 ret++;
229 reqsize = bhnext->b_size;
230 } else if (CONTIGUOUS_BUFFERS(bh, bhnext)) {
231 /*
232 * This one is OK. Let it go.
233 */
234 #ifdef DMA_SEGMENT_SIZE_LIMITED
235 /* Note scsi_malloc is only able to hand out
236 * chunks of memory in sizes of PAGE_SIZE or
237 * less. Thus we need to keep track of
238 * the size of the piece that we have
239 * seen so far, and if we have hit
240 * the limit of PAGE_SIZE, then we are
241 * kind of screwed and we need to start
242 * another segment.
243 */
244 if( dma_host
245 && virt_to_phys(bh->b_data) - 1 >= ISA_DMA_THRESHOLD
246 && reqsize + bhnext->b_size > PAGE_SIZE )
247 {
248 ret++;
249 reqsize = bhnext->b_size;
250 continue;
251 }
252 #endif
253 reqsize += bhnext->b_size;
254 continue;
255 }
256 ret++;
257 reqsize = bhnext->b_size;
258 } else {
259 ret++;
260 reqsize = bhnext->b_size;
261 }
262 }
263 if( remainder != NULL ) {
264 *remainder = reqsize;
265 }
266 return ret;
267 }
268
269 /*
270 * Function: recount_segments()
271 *
272 * Purpose: Recount the number of scatter-gather segments for this request.
273 *
274 * Arguments: req - request that needs recounting.
275 *
276 * Returns: Count of the number of SG segments for the request.
277 *
278 * Lock status: Irrelevant.
279 *
280 * Notes: This is only used when we have partially completed requests
281 * and the bit that is leftover is of an indeterminate size.
282 * This can come up if you get a MEDIUM_ERROR, for example,
283 * as we will have "completed" all of the sectors up to and
284 * including the bad sector, and the leftover bit is what
285 * we have to do now. This tends to be a rare occurrence, so
286 * we aren't busting our butts to instantiate separate versions
287 * of this function for the 4 different flag values. We
288 * probably should, however.
289 */
290 void
291 recount_segments(Scsi_Cmnd * SCpnt)
292 {
293 struct request *req;
294 struct Scsi_Host *SHpnt;
295 Scsi_Device * SDpnt;
296
297 req = &SCpnt->request;
298 SHpnt = SCpnt->host;
299 SDpnt = SCpnt->device;
300
301 req->nr_segments = __count_segments(req,
302 CLUSTERABLE_DEVICE(SHpnt, SDpnt),
303 SHpnt->unchecked_isa_dma, NULL);
304 }
305
306 #define MERGEABLE_BUFFERS(X,Y) \
307 (((((long)(X)->b_data+(X)->b_size)|((long)(Y)->b_data)) & \
308 (DMA_CHUNK_SIZE - 1)) == 0)
309
310 #ifdef DMA_CHUNK_SIZE
311 static inline int scsi_new_mergeable(request_queue_t * q,
312 struct request * req,
313 struct Scsi_Host *SHpnt,
314 int max_segments)
315 {
316 /*
317 * pci_map_sg will be able to merge these two
318 * into a single hardware sg entry, check if
319 * we'll have enough memory for the sg list.
320 * scsi.c allocates for this purpose
321 * min(64,sg_tablesize) entries.
322 */
323 if (req->nr_segments >= max_segments ||
324 req->nr_segments >= SHpnt->sg_tablesize)
325 return 0;
326 req->nr_segments++;
327 return 1;
328 }
329
330 static inline int scsi_new_segment(request_queue_t * q,
331 struct request * req,
332 struct Scsi_Host *SHpnt,
333 int max_segments)
334 {
335 /*
336 * pci_map_sg won't be able to map these two
337 * into a single hardware sg entry, so we have to
338 * check if things fit into sg_tablesize.
339 */
340 if (req->nr_hw_segments >= SHpnt->sg_tablesize ||
341 req->nr_segments >= SHpnt->sg_tablesize)
342 return 0;
343 req->nr_hw_segments++;
344 req->nr_segments++;
345 return 1;
346 }
347 #else
348 static inline int scsi_new_segment(request_queue_t * q,
349 struct request * req,
350 struct Scsi_Host *SHpnt,
351 int max_segments)
352 {
353 if (req->nr_segments < SHpnt->sg_tablesize &&
354 req->nr_segments < max_segments) {
355 /*
356 * This will form the start of a new segment. Bump the
357 * counter.
358 */
359 req->nr_segments++;
360 return 1;
361 } else {
362 return 0;
363 }
364 }
365 #endif
366
367 /*
368 * Function: __scsi_merge_fn()
369 *
370 * Purpose: Prototype for queue merge function.
371 *
372 * Arguments: q - Queue for which we are merging request.
373 * req - request into which we wish to merge.
374 * bh - Block which we may wish to merge into request
375 * use_clustering - 1 if this host wishes to use clustering
376 * dma_host - 1 if this host has ISA DMA issues (bus doesn't
377 * expose all of the address lines, so that DMA cannot
378 * be done from an arbitrary address).
379 *
380 * Returns: 1 if it is OK to merge the block into the request. 0
381 * if it is not OK.
382 *
383 * Lock status: io_request_lock is assumed to be held here.
384 *
385 * Notes: Some drivers have limited scatter-gather table sizes, and
386 * thus they cannot queue an infinitely large command. This
387 * function is called from ll_rw_blk before it attempts to merge
388 * a new block into a request to make sure that the request will
389 * not become too large.
390 *
391 * This function is not designed to be directly called. Instead
392 * it should be referenced from other functions where the
393 * use_clustering and dma_host parameters should be integer
394 * constants. The compiler should thus be able to properly
395 * optimize the code, eliminating stuff that is irrelevant.
396 * It is more maintainable to do this way with a single function
397 * than to have 4 separate functions all doing roughly the
398 * same thing.
399 */
400 __inline static int __scsi_back_merge_fn(request_queue_t * q,
401 struct request *req,
402 struct buffer_head *bh,
403 int max_segments,
404 int use_clustering,
405 int dma_host)
406 {
407 unsigned int count;
408 unsigned int segment_size = 0;
409 Scsi_Device *SDpnt;
410 struct Scsi_Host *SHpnt;
411
412 SDpnt = (Scsi_Device *) q->queuedata;
413 SHpnt = SDpnt->host;
414
415 #ifdef DMA_CHUNK_SIZE
416 if (max_segments > 64)
417 max_segments = 64;
418 #endif
419
420 if ((req->nr_sectors + (bh->b_size >> 9)) > SHpnt->max_sectors)
421 return 0;
422
423 if (use_clustering) {
424 /*
425 * See if we can do this without creating another
426 * scatter-gather segment. In the event that this is a
427 * DMA capable host, make sure that a segment doesn't span
428 * the DMA threshold boundary.
429 */
430 if (dma_host &&
431 virt_to_phys(req->bhtail->b_data) - 1 == ISA_DMA_THRESHOLD) {
432 goto new_end_segment;
433 }
434 if (CONTIGUOUS_BUFFERS(req->bhtail, bh)) {
435 #ifdef DMA_SEGMENT_SIZE_LIMITED
436 if( dma_host
437 && virt_to_phys(bh->b_data) - 1 >= ISA_DMA_THRESHOLD ) {
438 segment_size = 0;
439 count = __count_segments(req, use_clustering, dma_host, &segment_size);
440 if( segment_size + bh->b_size > PAGE_SIZE ) {
441 goto new_end_segment;
442 }
443 }
444 #endif
445 /*
446 * This one is OK. Let it go.
447 */
448 return 1;
449 }
450 }
451 new_end_segment:
452 #ifdef DMA_CHUNK_SIZE
453 if (MERGEABLE_BUFFERS(req->bhtail, bh))
454 return scsi_new_mergeable(q, req, SHpnt, max_segments);
455 #endif
456 return scsi_new_segment(q, req, SHpnt, max_segments);
457 }
458
459 __inline static int __scsi_front_merge_fn(request_queue_t * q,
460 struct request *req,
461 struct buffer_head *bh,
462 int max_segments,
463 int use_clustering,
464 int dma_host)
465 {
466 unsigned int count;
467 unsigned int segment_size = 0;
468 Scsi_Device *SDpnt;
469 struct Scsi_Host *SHpnt;
470
471 SDpnt = (Scsi_Device *) q->queuedata;
472 SHpnt = SDpnt->host;
473
474 #ifdef DMA_CHUNK_SIZE
475 if (max_segments > 64)
476 max_segments = 64;
477 #endif
478
479 if ((req->nr_sectors + (bh->b_size >> 9)) > SHpnt->max_sectors)
480 return 0;
481
482 if (use_clustering) {
483 /*
484 * See if we can do this without creating another
485 * scatter-gather segment. In the event that this is a
486 * DMA capable host, make sure that a segment doesn't span
487 * the DMA threshold boundary.
488 */
489 if (dma_host &&
490 virt_to_phys(bh->b_data) - 1 == ISA_DMA_THRESHOLD) {
491 goto new_start_segment;
492 }
493 if (CONTIGUOUS_BUFFERS(bh, req->bh)) {
494 #ifdef DMA_SEGMENT_SIZE_LIMITED
495 if( dma_host
496 && virt_to_phys(bh->b_data) - 1 >= ISA_DMA_THRESHOLD ) {
497 segment_size = bh->b_size;
498 count = __count_segments(req, use_clustering, dma_host, &segment_size);
499 if( count != req->nr_segments ) {
500 goto new_start_segment;
501 }
502 }
503 #endif
504 /*
505 * This one is OK. Let it go.
506 */
507 return 1;
508 }
509 }
510 new_start_segment:
511 #ifdef DMA_CHUNK_SIZE
512 if (MERGEABLE_BUFFERS(bh, req->bh))
513 return scsi_new_mergeable(q, req, SHpnt, max_segments);
514 #endif
515 return scsi_new_segment(q, req, SHpnt, max_segments);
516 }
517
518 /*
519 * Function: scsi_merge_fn_()
520 *
521 * Purpose: queue merge function.
522 *
523 * Arguments: q - Queue for which we are merging request.
524 * req - request into which we wish to merge.
525 * bh - Block which we may wish to merge into request
526 *
527 * Returns: 1 if it is OK to merge the block into the request. 0
528 * if it is not OK.
529 *
530 * Lock status: io_request_lock is assumed to be held here.
531 *
532 * Notes: Optimized for different cases depending upon whether
533 * ISA DMA is in use and whether clustering should be used.
534 */
535 #define MERGEFCT(_FUNCTION, _BACK_FRONT, _CLUSTER, _DMA) \
536 static int _FUNCTION(request_queue_t * q, \
537 struct request * req, \
538 struct buffer_head * bh, \
539 int max_segments) \
540 { \
541 int ret; \
542 SANITY_CHECK(req, _CLUSTER, _DMA); \
543 ret = __scsi_ ## _BACK_FRONT ## _merge_fn(q, \
544 req, \
545 bh, \
546 max_segments, \
547 _CLUSTER, \
548 _DMA); \
549 return ret; \
550 }
551
552 /* Version with use_clustering 0 and dma_host 1 is not necessary,
553 * since the only use of dma_host above is protected by use_clustering.
554 */
555 MERGEFCT(scsi_back_merge_fn_, back, 0, 0)
556 MERGEFCT(scsi_back_merge_fn_c, back, 1, 0)
557 MERGEFCT(scsi_back_merge_fn_dc, back, 1, 1)
558
559 MERGEFCT(scsi_front_merge_fn_, front, 0, 0)
560 MERGEFCT(scsi_front_merge_fn_c, front, 1, 0)
561 MERGEFCT(scsi_front_merge_fn_dc, front, 1, 1)
562
563 /*
564 * Function: __scsi_merge_requests_fn()
565 *
566 * Purpose: Prototype for queue merge function.
567 *
568 * Arguments: q - Queue for which we are merging request.
569 * req - request into which we wish to merge.
570 * next - 2nd request that we might want to combine with req
571 * use_clustering - 1 if this host wishes to use clustering
572 * dma_host - 1 if this host has ISA DMA issues (bus doesn't
573 * expose all of the address lines, so that DMA cannot
574 * be done from an arbitrary address).
575 *
576 * Returns: 1 if it is OK to merge the two requests. 0
577 * if it is not OK.
578 *
579 * Lock status: io_request_lock is assumed to be held here.
580 *
581 * Notes: Some drivers have limited scatter-gather table sizes, and
582 * thus they cannot queue an infinitely large command. This
583 * function is called from ll_rw_blk before it attempts to merge
584 * a new block into a request to make sure that the request will
585 * not become too large.
586 *
587 * This function is not designed to be directly called. Instead
588 * it should be referenced from other functions where the
589 * use_clustering and dma_host parameters should be integer
590 * constants. The compiler should thus be able to properly
591 * optimize the code, eliminating stuff that is irrelevant.
592 * It is more maintainable to do this way with a single function
593 * than to have 4 separate functions all doing roughly the
594 * same thing.
595 */
596 __inline static int __scsi_merge_requests_fn(request_queue_t * q,
597 struct request *req,
598 struct request *next,
599 int max_segments,
600 int use_clustering,
601 int dma_host)
602 {
603 Scsi_Device *SDpnt;
604 struct Scsi_Host *SHpnt;
605
606 /*
607 * First check if the either of the requests are re-queued
608 * requests. Can't merge them if they are.
609 */
610 if (req->special || next->special)
611 return 0;
612
613 SDpnt = (Scsi_Device *) q->queuedata;
614 SHpnt = SDpnt->host;
615
616 #ifdef DMA_CHUNK_SIZE
617 if (max_segments > 64)
618 max_segments = 64;
619
620 /* If it would not fit into prepared memory space for sg chain,
621 * then don't allow the merge.
622 */
623 if (req->nr_segments + next->nr_segments - 1 > max_segments ||
624 req->nr_segments + next->nr_segments - 1 > SHpnt->sg_tablesize) {
625 return 0;
626 }
627 if (req->nr_hw_segments + next->nr_hw_segments - 1 > SHpnt->sg_tablesize) {
628 return 0;
629 }
630 #else
631 /*
632 * If the two requests together are too large (even assuming that we
633 * can merge the boundary requests into one segment, then don't
634 * allow the merge.
635 */
636 if (req->nr_segments + next->nr_segments - 1 > SHpnt->sg_tablesize) {
637 return 0;
638 }
639 #endif
640
641 if ((req->nr_sectors + next->nr_sectors) > SHpnt->max_sectors)
642 return 0;
643
644 /*
645 * The main question is whether the two segments at the boundaries
646 * would be considered one or two.
647 */
648 if (use_clustering) {
649 /*
650 * See if we can do this without creating another
651 * scatter-gather segment. In the event that this is a
652 * DMA capable host, make sure that a segment doesn't span
653 * the DMA threshold boundary.
654 */
655 if (dma_host &&
656 virt_to_phys(req->bhtail->b_data) - 1 == ISA_DMA_THRESHOLD) {
657 goto dont_combine;
658 }
659 #ifdef DMA_SEGMENT_SIZE_LIMITED
660 /*
661 * We currently can only allocate scatter-gather bounce
662 * buffers in chunks of PAGE_SIZE or less.
663 */
664 if (dma_host
665 && CONTIGUOUS_BUFFERS(req->bhtail, next->bh)
666 && virt_to_phys(req->bhtail->b_data) - 1 >= ISA_DMA_THRESHOLD )
667 {
668 int segment_size = 0;
669 int count = 0;
670
671 count = __count_segments(req, use_clustering, dma_host, &segment_size);
672 count += __count_segments(next, use_clustering, dma_host, &segment_size);
673 if( count != req->nr_segments + next->nr_segments ) {
674 goto dont_combine;
675 }
676 }
677 #endif
678 if (CONTIGUOUS_BUFFERS(req->bhtail, next->bh)) {
679 /*
680 * This one is OK. Let it go.
681 */
682 req->nr_segments += next->nr_segments - 1;
683 #ifdef DMA_CHUNK_SIZE
684 req->nr_hw_segments += next->nr_hw_segments - 1;
685 #endif
686 return 1;
687 }
688 }
689 dont_combine:
690 #ifdef DMA_CHUNK_SIZE
691 if (req->nr_segments + next->nr_segments > max_segments ||
692 req->nr_segments + next->nr_segments > SHpnt->sg_tablesize) {
693 return 0;
694 }
695 /* If dynamic DMA mapping can merge last segment in req with
696 * first segment in next, then the check for hw segments was
697 * done above already, so we can always merge.
698 */
699 if (MERGEABLE_BUFFERS (req->bhtail, next->bh)) {
700 req->nr_hw_segments += next->nr_hw_segments - 1;
701 } else if (req->nr_hw_segments + next->nr_hw_segments > SHpnt->sg_tablesize) {
702 return 0;
703 } else {
704 req->nr_hw_segments += next->nr_hw_segments;
705 }
706 req->nr_segments += next->nr_segments;
707 return 1;
708 #else
709 /*
710 * We know that the two requests at the boundary should not be combined.
711 * Make sure we can fix something that is the sum of the two.
712 * A slightly stricter test than we had above.
713 */
714 if (req->nr_segments + next->nr_segments > max_segments ||
715 req->nr_segments + next->nr_segments > SHpnt->sg_tablesize) {
716 return 0;
717 } else {
718 /*
719 * This will form the start of a new segment. Bump the
720 * counter.
721 */
722 req->nr_segments += next->nr_segments;
723 return 1;
724 }
725 #endif
726 }
727
728 /*
729 * Function: scsi_merge_requests_fn_()
730 *
731 * Purpose: queue merge function.
732 *
733 * Arguments: q - Queue for which we are merging request.
734 * req - request into which we wish to merge.
735 * bh - Block which we may wish to merge into request
736 *
737 * Returns: 1 if it is OK to merge the block into the request. 0
738 * if it is not OK.
739 *
740 * Lock status: io_request_lock is assumed to be held here.
741 *
742 * Notes: Optimized for different cases depending upon whether
743 * ISA DMA is in use and whether clustering should be used.
744 */
745 #define MERGEREQFCT(_FUNCTION, _CLUSTER, _DMA) \
746 static int _FUNCTION(request_queue_t * q, \
747 struct request * req, \
748 struct request * next, \
749 int max_segments) \
750 { \
751 int ret; \
752 SANITY_CHECK(req, _CLUSTER, _DMA); \
753 ret = __scsi_merge_requests_fn(q, req, next, max_segments, _CLUSTER, _DMA); \
754 return ret; \
755 }
756
757 /* Version with use_clustering 0 and dma_host 1 is not necessary,
758 * since the only use of dma_host above is protected by use_clustering.
759 */
760 MERGEREQFCT(scsi_merge_requests_fn_, 0, 0)
761 MERGEREQFCT(scsi_merge_requests_fn_c, 1, 0)
762 MERGEREQFCT(scsi_merge_requests_fn_dc, 1, 1)
763 /*
764 * Function: __init_io()
765 *
766 * Purpose: Prototype for io initialize function.
767 *
768 * Arguments: SCpnt - Command descriptor we wish to initialize
769 * sg_count_valid - 1 if the sg count in the req is valid.
770 * use_clustering - 1 if this host wishes to use clustering
771 * dma_host - 1 if this host has ISA DMA issues (bus doesn't
772 * expose all of the address lines, so that DMA cannot
773 * be done from an arbitrary address).
774 *
775 * Returns: 1 on success.
776 *
777 * Lock status:
778 *
779 * Notes: Only the SCpnt argument should be a non-constant variable.
780 * This function is designed in such a way that it will be
781 * invoked from a series of small stubs, each of which would
782 * be optimized for specific circumstances.
783 *
784 * The advantage of this is that hosts that don't do DMA
785 * get versions of the function that essentially don't have
786 * any of the DMA code. Same goes for clustering - in the
787 * case of hosts with no need for clustering, there is no point
788 * in a whole bunch of overhead.
789 *
790 * Finally, in the event that a host has set can_queue to SG_ALL
791 * implying that there is no limit to the length of a scatter
792 * gather list, the sg count in the request won't be valid
793 * (mainly because we don't need queue management functions
794 * which keep the tally uptodate.
795 */
796 __inline static int __init_io(Scsi_Cmnd * SCpnt,
797 int sg_count_valid,
798 int use_clustering,
799 int dma_host)
800 {
801 struct buffer_head * bh;
802 struct buffer_head * bhprev;
803 char * buff;
804 int count;
805 int i;
806 struct request * req;
807 int sectors;
808 struct scatterlist * sgpnt;
809 int this_count;
810
811 /*
812 * FIXME(eric) - don't inline this - it doesn't depend on the
813 * integer flags. Come to think of it, I don't think this is even
814 * needed any more. Need to play with it and see if we hit the
815 * panic. If not, then don't bother.
816 */
817 if (!SCpnt->request.bh) {
818 /*
819 * Case of page request (i.e. raw device), or unlinked buffer
820 * Typically used for swapping, but this isn't how we do
821 * swapping any more.
822 */
823 panic("I believe this is dead code. If we hit this, I was wrong");
824 #if 0
825 SCpnt->request_bufflen = SCpnt->request.nr_sectors << 9;
826 SCpnt->request_buffer = SCpnt->request.buffer;
827 SCpnt->use_sg = 0;
828 /*
829 * FIXME(eric) - need to handle DMA here.
830 */
831 #endif
832 return 1;
833 }
834 req = &SCpnt->request;
835 /*
836 * First we need to know how many scatter gather segments are needed.
837 */
838 if (!sg_count_valid) {
839 count = __count_segments(req, use_clustering, dma_host, NULL);
840 } else {
841 count = req->nr_segments;
842 }
843
844 /*
845 * If the dma pool is nearly empty, then queue a minimal request
846 * with a single segment. Typically this will satisfy a single
847 * buffer.
848 */
849 if (dma_host && scsi_dma_free_sectors <= 10) {
850 this_count = SCpnt->request.current_nr_sectors;
851 goto single_segment;
852 }
853 /*
854 * Don't bother with scatter-gather if there is only one segment.
855 */
856 if (count == 1) {
857 this_count = SCpnt->request.nr_sectors;
858 goto single_segment;
859 }
860 SCpnt->use_sg = count;
861
862 /*
863 * Allocate the actual scatter-gather table itself.
864 * scsi_malloc can only allocate in chunks of 512 bytes
865 */
866 SCpnt->sglist_len = (SCpnt->use_sg
867 * sizeof(struct scatterlist) + 511) & ~511;
868
869 sgpnt = (struct scatterlist *) scsi_malloc(SCpnt->sglist_len);
870
871 /*
872 * Now fill the scatter-gather table.
873 */
874 if (!sgpnt) {
875 /*
876 * If we cannot allocate the scatter-gather table, then
877 * simply write the first buffer all by itself.
878 */
879 printk("Warning - running *really* short on DMA buffers\n");
880 this_count = SCpnt->request.current_nr_sectors;
881 goto single_segment;
882 }
883 /*
884 * Next, walk the list, and fill in the addresses and sizes of
885 * each segment.
886 */
887 memset(sgpnt, 0, SCpnt->sglist_len);
888 SCpnt->request_buffer = (char *) sgpnt;
889 SCpnt->request_bufflen = 0;
890 bhprev = NULL;
891
892 for (count = 0, bh = SCpnt->request.bh;
893 bh; bh = bh->b_reqnext) {
894 if (use_clustering && bhprev != NULL) {
895 if (dma_host &&
896 virt_to_phys(bhprev->b_data) - 1 == ISA_DMA_THRESHOLD) {
897 /* Nothing - fall through */
898 } else if (CONTIGUOUS_BUFFERS(bhprev, bh)) {
899 /*
900 * This one is OK. Let it go. Note that we
901 * do not have the ability to allocate
902 * bounce buffer segments > PAGE_SIZE, so
903 * for now we limit the thing.
904 */
905 if( dma_host ) {
906 #ifdef DMA_SEGMENT_SIZE_LIMITED
907 if( virt_to_phys(bh->b_data) - 1 < ISA_DMA_THRESHOLD
908 || sgpnt[count - 1].length + bh->b_size <= PAGE_SIZE ) {
909 sgpnt[count - 1].length += bh->b_size;
910 bhprev = bh;
911 continue;
912 }
913 #else
914 sgpnt[count - 1].length += bh->b_size;
915 bhprev = bh;
916 continue;
917 #endif
918 } else {
919 sgpnt[count - 1].length += bh->b_size;
920 SCpnt->request_bufflen += bh->b_size;
921 bhprev = bh;
922 continue;
923 }
924 }
925 }
926 count++;
927 sgpnt[count - 1].address = bh->b_data;
928 sgpnt[count - 1].length += bh->b_size;
929 if (!dma_host) {
930 SCpnt->request_bufflen += bh->b_size;
931 }
932 bhprev = bh;
933 }
934
935 /*
936 * Verify that the count is correct.
937 */
938 if (count != SCpnt->use_sg) {
939 printk("Incorrect number of segments after building list\n");
940 #ifdef CONFIG_SCSI_DEBUG_QUEUES
941 dump_stats(req, use_clustering, dma_host, count);
942 #endif
943 }
944 if (!dma_host) {
945 return 1;
946 }
947 /*
948 * Now allocate bounce buffers, if needed.
949 */
950 SCpnt->request_bufflen = 0;
951 for (i = 0; i < count; i++) {
952 sectors = (sgpnt[i].length >> 9);
953 SCpnt->request_bufflen += sgpnt[i].length;
954 if (virt_to_phys(sgpnt[i].address) + sgpnt[i].length - 1 >
955 ISA_DMA_THRESHOLD) {
956 if( scsi_dma_free_sectors - sectors <= 10 ) {
957 /*
958 * If this would nearly drain the DMA
959 * pool, mpty, then let's stop here.
960 * Don't make this request any larger.
961 * This is kind of a safety valve that
962 * we use - we could get screwed later
963 * on if we run out completely.
964 */
965 SCpnt->request_bufflen -= sgpnt[i].length;
966 SCpnt->use_sg = i;
967 if (i == 0) {
968 goto big_trouble;
969 }
970 break;
971 }
972
973 sgpnt[i].alt_address = sgpnt[i].address;
974 sgpnt[i].address =
975 (char *) scsi_malloc(sgpnt[i].length);
976 /*
977 * If we cannot allocate memory for this DMA bounce
978 * buffer, then queue just what we have done so far.
979 */
980 if (sgpnt[i].address == NULL) {
981 printk("Warning - running low on DMA memory\n");
982 SCpnt->request_bufflen -= sgpnt[i].length;
983 SCpnt->use_sg = i;
984 if (i == 0) {
985 goto big_trouble;
986 }
987 break;
988 }
989 if (SCpnt->request.cmd == WRITE) {
990 memcpy(sgpnt[i].address, sgpnt[i].alt_address,
991 sgpnt[i].length);
992 }
993 }
994 }
995 return 1;
996
997 big_trouble:
998 /*
999 * We come here in the event that we get one humongous
1000 * request, where we need a bounce buffer, and the buffer is
1001 * more than we can allocate in a single call to
1002 * scsi_malloc(). In addition, we only come here when it is
1003 * the 0th element of the scatter-gather table that gets us
1004 * into this trouble. As a fallback, we fall back to
1005 * non-scatter-gather, and ask for a single segment. We make
1006 * a half-hearted attempt to pick a reasonably large request
1007 * size mainly so that we don't thrash the thing with
1008 * iddy-biddy requests.
1009 */
1010
1011 /*
1012 * The original number of sectors in the 0th element of the
1013 * scatter-gather table.
1014 */
1015 sectors = sgpnt[0].length >> 9;
1016
1017 /*
1018 * Free up the original scatter-gather table. Note that since
1019 * it was the 0th element that got us here, we don't have to
1020 * go in and free up memory from the other slots.
1021 */
1022 SCpnt->request_bufflen = 0;
1023 SCpnt->use_sg = 0;
1024 scsi_free(SCpnt->request_buffer, SCpnt->sglist_len);
1025
1026 /*
1027 * Make an attempt to pick up as much as we reasonably can.
1028 * Just keep adding sectors until the pool starts running kind of
1029 * low. The limit of 30 is somewhat arbitrary - the point is that
1030 * it would kind of suck if we dropped down and limited ourselves to
1031 * single-block requests if we had hundreds of free sectors.
1032 */
1033 if( scsi_dma_free_sectors > 30 ) {
1034 for (this_count = 0, bh = SCpnt->request.bh;
1035 bh; bh = bh->b_reqnext) {
1036 if( scsi_dma_free_sectors - this_count < 30
1037 || this_count == sectors )
1038 {
1039 break;
1040 }
1041 this_count += bh->b_size >> 9;
1042 }
1043
1044 } else {
1045 /*
1046 * Yow! Take the absolute minimum here.
1047 */
1048 this_count = SCpnt->request.current_nr_sectors;
1049 }
1050
1051 /*
1052 * Now drop through into the single-segment case.
1053 */
1054
1055 single_segment:
1056 /*
1057 * Come here if for any reason we choose to do this as a single
1058 * segment. Possibly the entire request, or possibly a small
1059 * chunk of the entire request.
1060 */
1061 bh = SCpnt->request.bh;
1062 buff = SCpnt->request.buffer;
1063
1064 if (dma_host) {
1065 /*
1066 * Allocate a DMA bounce buffer. If the allocation fails, fall
1067 * back and allocate a really small one - enough to satisfy
1068 * the first buffer.
1069 */
1070 if (virt_to_phys(SCpnt->request.bh->b_data)
1071 + (this_count << 9) - 1 > ISA_DMA_THRESHOLD) {
1072 buff = (char *) scsi_malloc(this_count << 9);
1073 if (!buff) {
1074 printk("Warning - running low on DMA memory\n");
1075 this_count = SCpnt->request.current_nr_sectors;
1076 buff = (char *) scsi_malloc(this_count << 9);
1077 if (!buff) {
1078 dma_exhausted(SCpnt, 0);
1079 }
1080 }
1081 if (SCpnt->request.cmd == WRITE)
1082 memcpy(buff, (char *) SCpnt->request.buffer, this_count << 9);
1083 }
1084 }
1085 SCpnt->request_bufflen = this_count << 9;
1086 SCpnt->request_buffer = buff;
1087 SCpnt->use_sg = 0;
1088 return 1;
1089 }
1090
1091 #define INITIO(_FUNCTION, _VALID, _CLUSTER, _DMA) \
1092 static int _FUNCTION(Scsi_Cmnd * SCpnt) \
1093 { \
1094 return __init_io(SCpnt, _VALID, _CLUSTER, _DMA); \
1095 }
1096
1097 /*
1098 * ll_rw_blk.c now keeps track of the number of segments in
1099 * a request. Thus we don't have to do it any more here.
1100 * We always force "_VALID" to 1. Eventually clean this up
1101 * and get rid of the extra argument.
1102 */
1103 INITIO(scsi_init_io_v, 1, 0, 0)
1104 INITIO(scsi_init_io_vd, 1, 0, 1)
1105 INITIO(scsi_init_io_vc, 1, 1, 0)
1106 INITIO(scsi_init_io_vdc, 1, 1, 1)
1107
1108 /*
1109 * Function: initialize_merge_fn()
1110 *
1111 * Purpose: Initialize merge function for a host
1112 *
1113 * Arguments: SHpnt - Host descriptor.
1114 *
1115 * Returns: Nothing.
1116 *
1117 * Lock status:
1118 *
1119 * Notes:
1120 */
1121 void initialize_merge_fn(Scsi_Device * SDpnt)
1122 {
1123 request_queue_t *q;
1124 struct Scsi_Host *SHpnt;
1125 SHpnt = SDpnt->host;
1126
1127 q = &SDpnt->request_queue;
1128
1129 /*
1130 * If the host has already selected a merge manager, then don't
1131 * pick a new one.
1132 */
1133 #if 0
1134 if (q->back_merge_fn && q->front_merge_fn)
1135 return;
1136 #endif
1137 /*
1138 * If this host has an unlimited tablesize, then don't bother with a
1139 * merge manager. The whole point of the operation is to make sure
1140 * that requests don't grow too large, and this host isn't picky.
1141 *
1142 * Note that ll_rw_blk.c is effectively maintaining a segment
1143 * count which is only valid if clustering is used, and it obviously
1144 * doesn't handle the DMA case. In the end, it
1145 * is simply easier to do it ourselves with our own functions
1146 * rather than rely upon the default behavior of ll_rw_blk.
1147 */
1148 if (!CLUSTERABLE_DEVICE(SHpnt, SDpnt) && SHpnt->unchecked_isa_dma == 0) {
1149 q->back_merge_fn = scsi_back_merge_fn_;
1150 q->front_merge_fn = scsi_front_merge_fn_;
1151 q->merge_requests_fn = scsi_merge_requests_fn_;
1152 SDpnt->scsi_init_io_fn = scsi_init_io_v;
1153 } else if (!CLUSTERABLE_DEVICE(SHpnt, SDpnt) && SHpnt->unchecked_isa_dma != 0) {
1154 q->back_merge_fn = scsi_back_merge_fn_;
1155 q->front_merge_fn = scsi_front_merge_fn_;
1156 q->merge_requests_fn = scsi_merge_requests_fn_;
1157 SDpnt->scsi_init_io_fn = scsi_init_io_vd;
1158 } else if (CLUSTERABLE_DEVICE(SHpnt, SDpnt) && SHpnt->unchecked_isa_dma == 0) {
1159 q->back_merge_fn = scsi_back_merge_fn_c;
1160 q->front_merge_fn = scsi_front_merge_fn_c;
1161 q->merge_requests_fn = scsi_merge_requests_fn_c;
1162 SDpnt->scsi_init_io_fn = scsi_init_io_vc;
1163 } else if (CLUSTERABLE_DEVICE(SHpnt, SDpnt) && SHpnt->unchecked_isa_dma != 0) {
1164 q->back_merge_fn = scsi_back_merge_fn_dc;
1165 q->front_merge_fn = scsi_front_merge_fn_dc;
1166 q->merge_requests_fn = scsi_merge_requests_fn_dc;
1167 SDpnt->scsi_init_io_fn = scsi_init_io_vdc;
1168 }
1169 }
1170